/****************************************************************************
*
*    Copyright (c) 2020 Vivante Corporation
*
*    Permission is hereby granted, free of charge, to any person obtaining a
*    copy of this software and associated documentation files (the "Software"),
*    to deal in the Software without restriction, including without limitation
*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
*    and/or sell copies of the Software, and to permit persons to whom the
*    Software is furnished to do so, subject to the following conditions:
*
*    The above copyright notice and this permission notice shall be included in
*    all copies or substantial portions of the Software.
*
*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
*    DEALINGS IN THE SOFTWARE.
*
*****************************************************************************/

/* WARNING! AUTO-GENERATED, DO NOT MODIFY MANUALLY */

#include <stdlib.h>
#include <string.h>
#include "vsi_nn_prv.h"
#include "utils/vsi_nn_util.h"
#include "kernel/vsi_nn_kernel.h"
#include "libnnext/vsi_nn_libnnext_resource.h"


static const char a_times_b_plus_c_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniA_Times_B_2x8;\n\
_viv_uniform VXC_512Bits uniA_Plus_B_2x8;\n\
__kernel void a_times_b_plus_c_F16_F16_F16toF16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_half8   src0, src1, src2, dst;\n\
    vxc_ushort8 vec0, vec1, vec2, result;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage2DArray(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void a_times_b_plus_c_F16_F16_F16toF16_2D\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_half8   src0, src1, src2, dst;\n\
    vxc_ushort8 vec0, vec1, vec2, result;\n\
\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniA_Times_B_lo_4x4;\n\
_viv_uniform VXC_512Bits uniA_Times_B_hi_4x4;\n\
__kernel void a_times_b_plus_c_F16_F16_F32toF16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_half8   src0, src1, dst;\n\
    vxc_ushort8 vec0, vec1, result;\n\
    float4 b0, b1;\n\
    float4 dst0, dst1;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    b0 = read_imagef(input2, coord);\n\
    coord.x += 4;\n\
    b1 = read_imagef(input2, coord);\n\
    coord.x -= 4;\n\
\n\
    VXC_DP4x4(dst0, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_lo_4x4);\n\
    VXC_DP4x4(dst1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_hi_4x4);\n\
    dst0 += b0;\n\
    dst1 += b1;\n\
\n\
    half4 t0, t1;\n\
    _viv_asm(CONV, t0, dst0);\n\
    _viv_asm(CONV, t1, dst1);\n\
    VXC_DP2x8(dst, t0, t1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void a_times_b_plus_c_F16_F16_F32toF16_2D\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_t         input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_half8   src0, src1, dst;\n\
    vxc_ushort8 vec0, vec1, result;\n\
    float4 b0, b1;\n\
    float4 dst0, dst1;\n\
\n\
    VXC_ReadImage(vec0, input0, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    b0 = read_imagef(input2, coord.xy);\n\
    coord.z = coord.x + 4;\n\
    b1 = read_imagef(input2, coord.zy);\n\
\n\
    VXC_DP4x4(dst0, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_lo_4x4);\n\
    VXC_DP4x4(dst1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_hi_4x4);\n\
    dst0 += b0;\n\
    dst1 += b1;\n\
\n\
    half4 t0, t1;\n\
    _viv_asm(CONV, t0, dst0);\n\
    _viv_asm(CONV, t1, dst1);\n\
    VXC_DP2x8(dst, t0, t1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of a_times_b_plus_c_vx*/

static const char add_mean_std_norm_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************Tensor add mean stddev norm float16*********************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform float rsEps;\n\
_viv_uniform VXC_512Bits uniAddFp16_2x8;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniAddFp16toFp32Lo_4x4;\n\
_viv_uniform VXC_512Bits uniAddFp16toFp32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
// one group(16 threads) calculates one row\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_F16_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    float eps)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0, src1, src2;\n\
    float pSum = 0, pSqr = 0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_half8 in_h, in_h1, in_h2;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, in_h1, src1, 16);\n\
        VXC_DP2x8(in_h2, in_h, in_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16_2x8);\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, in_h2, in_h2, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        pSum += sumsqr.x;\n\
        pSqr += sumsqr.y;\n\
    }\n\
\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, in_h1, src1, 16);\n\
\n\
        vxc_float4 in_f0, in_f1;\n\
        VXC_DP4x4(in_f0, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
               uniAddFp16toFp32Lo_4x4);\n\
        VXC_DP4x4(in_f1, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16toFp32Hi_4x4);\n\
\n\
        vxc_float4 norm0, norm1;\n\
        half4 norm_h0, norm_h1;\n\
\n\
        norm0 = in_f0 * stddev_inv + rMeanStd;\n\
        norm1 = in_f1 * stddev_inv + rMeanStd;\n\
        _viv_asm(CONV, norm_h0, norm0);\n\
        _viv_asm(CONV, norm_h1, norm1);\n\
\n\
        VXC_DP2x8(src2, norm_h0, norm_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_1_Lo_2x8;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_U8_U8toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    float eps)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_uchar8 src0, src1;\n\
    vxc_short8 src2;\n\
    float pSum = 0, pSqr = 0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_half8 in_h, in_h1, in_h2;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    vxc_ushort8 ms0, ms1;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, ms1, multAndoutZP1, 16);\n\
\n\
    for(; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(in_h, src0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
        VXC_DP2x8(in_h1, src1, ms1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_1_Lo_2x8);\n\
        VXC_DP2x8(in_h2, in_h, in_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16_2x8);\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, in_h2, in_h2, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        pSum += sumsqr.x;\n\
        pSqr += sumsqr.y;\n\
    }\n\
\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(in_h, src0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
        VXC_DP2x8(in_h1, src1, ms1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_1_Lo_2x8);\n\
\n\
        vxc_float4 in_f0, in_f1;\n\
        VXC_DP4x4(in_f0, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
               uniAddFp16toFp32Lo_4x4);\n\
        VXC_DP4x4(in_f1, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16toFp32Hi_4x4);\n\
\n\
        vxc_float4 norm0, norm1;\n\
        half4 norm_h0, norm_h1;\n\
\n\
        norm0 = in_f0 * stddev_inv + rMeanStd;\n\
        norm1 = in_f1 * stddev_inv + rMeanStd;\n\
        _viv_asm(CONV, norm_h0, norm0);\n\
        _viv_asm(CONV, norm_h1, norm1);\n\
\n\
        VXC_DP2x8(src2, norm_h0, norm_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16ScaleToFp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16ScaleToFp32Sec_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Secd_4x4;\n\
_viv_uniform float inScale_i16;\n\
_viv_uniform float inScale1_i16;\n\
\n\
// one group(16 threads) calculates one row\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_I16_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    float eps)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0, src1, src2;\n\
    float pSum = 0, pSqr = 0;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    half scale_h, scale_h1;\n\
    _viv_asm(CONV, scale_h, inScale_i16);\n\
    _viv_asm(CONV, scale_h1, inScale1_i16);\n\
    float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3;\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
\n\
    for(; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(tmpVal0, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal1, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        VXC_DP4x4(tmpVal2, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal3, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        tmpVal0 += tmpVal2;\n\
        tmpVal1 += tmpVal3;\n\
\n\
        vxc_float4 sumsqr;\n\
        sumsqr = tmpVal0 * tmpVal0 + tmpVal1 * tmpVal1; // sqr\n\
        tmpVal2 = tmpVal0 + tmpVal1; // pre sum\n\
\n\
        pSum += dot(tmpVal2, one);\n\
        pSqr += dot(sumsqr, one);\n\
    }\n\
\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 data0;\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(tmpVal0, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal1, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        VXC_DP4x4(tmpVal2, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal3, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        tmpVal0 += tmpVal2;\n\
        tmpVal1 += tmpVal3;\n\
\n\
        vxc_float4 norm0, norm1;\n\
        half4 norm_h0, norm_h1;\n\
\n\
        norm0 = tmpVal0 * stddev_inv + rMeanStd;\n\
        norm1 = tmpVal1 * stddev_inv + rMeanStd;\n\
        _viv_asm(CONV, norm_h0, norm0);\n\
        _viv_asm(CONV, norm_h1, norm1);\n\
\n\
        VXC_DP2x8(src2, norm_h0, norm_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of add_mean_std_norm_vx*/

static const char argmax_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int inputWidth;\n\
_viv_uniform VXC_512Bits uniPackedIdxAddSat_2x8;\n\
_viv_uniform VXC_512Bits uniSrcT2DstT_2x8;\n\
\n\
#define TENSOR_ARGMAX_AXIS0_8BITS(src_type_name, dst_type_name, src_type, \\\n\
                                cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmax_axis0_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type maxValue, maxVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage2DArray(maxVec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(value, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value > maxVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        maxVec = max(maxVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMax3_Integer(maxValue, maxVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMax3_Integer(maxValue, maxValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    maxValue.s01234567 = maxValue.s00000000; \\\n\
 \\\n\
    cond_type1 _maxVal; \\\n\
    VXC_Clamp(_maxVal, maxVec, maxValue, maxValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _maxVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _maxVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS0_8BITS(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMAX_AXIS0_8BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmax_axis0_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    src_type maxValue, maxVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage(maxVec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(value, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value > maxVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        maxVec = max(maxVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMax3_Integer(maxValue, maxVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMax3_Integer(maxValue, maxValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    maxValue.s01234567 = maxValue.s00000000; \\\n\
 \\\n\
    cond_type1 _maxVal; \\\n\
    VXC_Clamp(_maxVal, maxVec, maxValue, maxValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _maxVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _maxVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalf2Float32_4x4;\n\
\n\
#define TENSOR_ARGMAX_AXIS0_F16_2D(dst_type_name, dst_type) \\\n\
__kernel void argmax_axis0_F16to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 maxValue, value; \\\n\
 \\\n\
    VXC_ReadImage(vec0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(maxValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(vec1, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value > maxValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        maxValue = value > maxValue ? value : maxValue; \\\n\
    } \\\n\
 \\\n\
    float4 maxVec; \\\n\
    float2 maxVal2 = maxValue.xy > maxValue.zw ? maxValue.xy : maxValue.zw; \\\n\
    maxVec.x = maxVal2.x > maxVal2.y ? maxVal2.x : maxVal2.y; \\\n\
    int4 condition; \\\n\
    condition = maxVec.xxxx == maxValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS0_F16_2D(I16, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_F16_2D(U8,  vxc_uchar8)\n\
\n\
\n\
#define TENSOR_ARGMAX_AXIS0_F16(dst_type_name, dst_type) \\\n\
__kernel void argmax_axis0_F16to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 maxValue, value; \\\n\
 \\\n\
    VXC_ReadImage2DArray(vec0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(maxValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(vec1, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value > maxValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        maxValue = value > maxValue ? value : maxValue; \\\n\
    } \\\n\
 \\\n\
    float4 maxVec; \\\n\
    float2 maxVal2 = maxValue.xy > maxValue.zw ? maxValue.xy : maxValue.zw; \\\n\
    maxVec.x = maxVal2.x > maxVal2.y ? maxVal2.x : maxVal2.y; \\\n\
    int4 condition; \\\n\
    condition = maxVec.xxxx == maxValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS0_F16(I16, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_F16(U8,  vxc_uchar8)\n\
"; /* end of argmax_axis0_vx*/

static const char argmax_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
\n\
#define TENSOR_ARGMAX_AXIS1_16BITS(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmax_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    copy_type max; \\\n\
    VXC_ReadImage2DArray(max, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, maxVal, max, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMax3_##inst_type(maxVal, maxVal, maxVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, max, maxVal, 16); \\\n\
       axis = (max == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS1_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMAX_AXIS1_16BITS_2D(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
__kernel void argmax_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    copy_type max; \\\n\
    VXC_ReadImage(max, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, maxVal, max, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMax3_##inst_type(maxVal, maxVal, maxVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, max, maxVal, 16); \\\n\
       axis = (max == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMAX_AXIS1_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmax_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage2DArray(maxVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_8BITS(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMAX_AXIS1_8BITS(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
\n\
#define TENSOR_ARGMAX_AXIS1_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmax_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage(maxVal, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(src, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(U8,  U8,  vxc_uchar16, vxc_uchar16)"; /* end of argmax_axis1_vx*/

static const char argmax_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
_viv_uniform VXC_512Bits uniExtract1stU8toI16_2x8;\n\
_viv_uniform VXC_512Bits uniExtract2ndU8toI16_2x8;\n\
\n\
#define TENSOR_ARGMAX_AXIS2_16BITS(src_type_name, dst_type_name,\\\n\
                src_type, copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    copy_type max; \\\n\
    VXC_ReadImage2DArray(max, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, maxVal, max, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       VXC_VertMax3_##inst_type(maxVal, maxVal, maxVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, max, maxVal, 16); \\\n\
       axis = (max == vec) ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS2_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMAX_AXIS2_16BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
\n\
#define TENSOR_ARGMAX_AXIS2_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage2DArray(maxVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       src_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_8BITS(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMAX_AXIS2_8BITS(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
\n\
#define TENSOR_ARGMAX_AXIS2_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(I8,  U8,  vxc_char8,  vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(U8,  I16, vxc_uchar8, vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMAX_AXIS2_MIX(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage2DArray(maxVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_MIX(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_MIX(U8,  I16, vxc_uchar8, vxc_short8)\n\
\n\
#define TENSOR_ARGMAX_AXIS2_MIX_OPT(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name##_opt( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage2DArray(maxVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_uchar16 axis; \\\n\
    vxc_uchar16 packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       src_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
    vxc_short8 dst0, dst1; \\\n\
    VXC_DP2x8(dst0, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniExtract1stU8toI16_2x8); \\\n\
    VXC_DP2x8(dst1, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniExtract2ndU8toI16_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xy, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    VXC_WriteImage(output, coord.xy, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_MIX_OPT(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_MIX_OPT(U8,  I16, vxc_uchar16, vxc_short8)\n\
"; /* end of argmax_axis2_vx*/

static const char argmin_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int inputWidth;\n\
_viv_uniform VXC_512Bits uniPackedIdxAddSat_2x8;\n\
_viv_uniform VXC_512Bits uniSrcT2DstT_2x8;\n\
\n\
#define TENSOR_ARGMIN_AXIS0_8BITS(src_type_name, dst_type_name, src_type, \\\n\
                                cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmin_axis0_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type minValue, minVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage2DArray(minVec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(value, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value < minVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        minVec = min(minVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMin3_Integer(minValue, minVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(minValue, minValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    minValue.s01234567 = minValue.s00000000; \\\n\
 \\\n\
    cond_type1 _minVal; \\\n\
    VXC_Clamp(_minVal, minVec, minValue, minValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _minVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _minVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS0_8BITS(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMIN_AXIS0_8BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmin_axis0_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    src_type minValue, minVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage(minVec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(value, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value < minVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        minVec = min(minVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMin3_Integer(minValue, minVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(minValue, minValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    minValue.s01234567 = minValue.s00000000; \\\n\
 \\\n\
    cond_type1 _minVal; \\\n\
    VXC_Clamp(_minVal, minVec, minValue, minValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _minVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _minVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalf2Float32_4x4;\n\
\n\
#define TENSOR_ARGMIN_AXIS0_F16_2D(dst_type_name, dst_type) \\\n\
__kernel void argmin_axis0_F16to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 minValue, value; \\\n\
 \\\n\
    VXC_ReadImage(vec0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(minValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(vec1, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value < minValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        minValue = value < minValue ? value : minValue; \\\n\
    } \\\n\
 \\\n\
    float4 minVec; \\\n\
    float2 minVal2 = minValue.xy < minValue.zw ? minValue.xy : minValue.zw; \\\n\
    minVec.x = minVal2.x < minVal2.y ? minVal2.x : minVal2.y; \\\n\
    int4 condition; \\\n\
    condition = minVec.xxxx == minValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS0_F16_2D(I16, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_F16_2D(U8,  vxc_uchar8)\n\
\n\
\n\
#define TENSOR_ARGMIN_AXIS0_F16(dst_type_name, dst_type) \\\n\
__kernel void argmin_axis0_F16to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 minValue, value; \\\n\
 \\\n\
    VXC_ReadImage2DArray(vec0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(minValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(vec1, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value < minValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        minValue = value < minValue ? value : minValue; \\\n\
    } \\\n\
 \\\n\
    float4 minVec; \\\n\
    float2 minVal2 = minValue.xy < minValue.zw ? minValue.xy : minValue.zw; \\\n\
    minVec.x = minVal2.x < minVal2.y ? minVal2.x : minVal2.y; \\\n\
    int4 condition; \\\n\
    condition = minVec.xxxx == minValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS0_F16(I16, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_F16(U8,  vxc_uchar8)\n\
"; /* end of argmin_axis0_vx*/

static const char argmin_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
\n\
#define TENSOR_ARGMIN_AXIS1_16BITS(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    copy_type min; \\\n\
    VXC_ReadImage2DArray(min, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, minVal, min, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMin3_##inst_type(minVal, minVal, minVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, min, minVal, 16); \\\n\
       axis = (min == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS1_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMIN_AXIS1_16BITS_2D(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    copy_type min; \\\n\
    VXC_ReadImage(min, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, minVal, min, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMin3_##inst_type(minVal, minVal, minVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, min, minVal, 16); \\\n\
       axis = (min == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMIN_AXIS1_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmin_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    VXC_ReadImage2DArray(minVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       minVal = min(minVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, minVal, minVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_8BITS(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMIN_AXIS1_8BITS(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
\n\
#define TENSOR_ARGMIN_AXIS1_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmin_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    VXC_ReadImage(minVal, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(src, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       minVal = min(minVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, minVal, minVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
\n\
"; /* end of argmin_axis1_vx*/

static const char argmin_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
\n\
#define TENSOR_ARGMIN_AXIS2_16BITS(src_type_name, dst_type_name,\\\n\
                src_type, copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    copy_type min; \\\n\
    VXC_ReadImage2DArray(min, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, minVal, min, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       VXC_VertMin3_##inst_type(minVal, minVal, minVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, min, minVal, 16); \\\n\
       axis = (min == vec) ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS2_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMIN_AXIS2_16BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
\n\
#define TENSOR_ARGMIN_AXIS2_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    VXC_ReadImage2DArray(minVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       minVal = min(minVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, minVal, minVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_8BITS(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS(I8,  U8,  vxc_char8,  vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS2_8BITS(U8,  I16, vxc_uchar8, vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS(U8,  U8,  vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMIN_AXIS2_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(I8,  U8,  vxc_char8,  vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(U8,  I16, vxc_uchar8, vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_uchar8)\n\
"; /* end of argmin_axis2_vx*/

static const char batchnorm_single_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDatatoF32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoF32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define BATCH_NORM_SH_IMPL(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst1( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage2DArray(_gamma, Gamma, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst1_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage(_gamma, Gamma, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta = read_imagef(Beta, coord.xy); \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst0( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage2DArray(_gamma, Gamma, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta0 = read_imagef(Beta, coord); \\\n\
    coord.x += 4; \\\n\
    float4 beta1 = read_imagef(Beta, coord); \\\n\
    coord.x -= 4; \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst0_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    coord.z += 4; \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage(_gamma, Gamma, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta0 = read_imagef(Beta, coord.xy); \\\n\
    float4 beta1 = read_imagef(Beta, coord.zy); \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
"; /* end of batchnorm_single_vx*/

static const char batchnorm_single_f32_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDatatoF32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoF32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define BATCH_NORM_SH_IMPL(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst1( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    int4 coord_in = coord; \\\n\
    int depth = get_image_array_size(Gamma); \\\n\
    _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
    float4 gamma = read_imagef(Gamma, coord_in); \\\n\
    coord_in.z = coord.z; \\\n\
    depth = get_image_array_size(Beta); \\\n\
    _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
    float4 beta = read_imagef(Beta, coord_in); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    float4 gamma0 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    float4 gamma1 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst1_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    coord.z = coord.x + 4; \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    float4 gamma = read_imagef(Gamma, coord.xy); \\\n\
    float4 beta = read_imagef(Beta, coord.xy); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    float4 gamma0 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    float4 gamma1 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst0( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    int4 coord_in0 = coord; \\\n\
    int depth = get_image_array_size(Gamma); \\\n\
    _viv_asm(CLAMP0MAX, coord_in0.z, coord_in0.z, depth - 1); \\\n\
    float4 gamma0 = read_imagef(Gamma, coord_in0); \\\n\
    int4 coord_in1 = coord; \\\n\
    depth = get_image_array_size(Beta); \\\n\
    _viv_asm(CLAMP0MAX, coord_in1.z, coord_in1.z, depth - 1); \\\n\
    float4 beta0 = read_imagef(Beta, coord_in1); \\\n\
    coord_in0.x += 4; \\\n\
    coord_in1.x += 4; \\\n\
    float4 gamma1 = read_imagef(Gamma, coord_in0); \\\n\
    float4 beta1 = read_imagef(Beta, coord_in1); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    gamma0 = gamma0 * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    gamma1 = gamma1 * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst0_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    coord.z += 4; \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    float4 gamma0 = read_imagef(Gamma, coord.xy); \\\n\
    float4 gamma1 = read_imagef(Gamma, coord.zy); \\\n\
    float4 beta0 = read_imagef(Beta, coord.xy); \\\n\
    float4 beta1 = read_imagef(Beta, coord.zy); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    gamma0 = gamma0 * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    gamma1 = gamma1 * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
"; /* end of batchnorm_single_f32_vx*/

static const char bilinear_grid_sample_BF16_to_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_even_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_odd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniBF16toFp32_part0_2x8;\n\
_viv_uniform VXC_512Bits uniBF16toFp32_part1_2x8;\n\
\n\
#define GRID_SAMPLE_BF16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 top; \\\n\
    vxc_short8 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    vxc_ushort8 tmp, dst; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
        _viv_asm(COPY, right4, src, 16); \\\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
        _viv_asm(COPY, left4,  src, 16); \\\n\
        right4    -= left4; \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
        _viv_asm(COPY, right4, src, 16); \\\n\
        VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
        _viv_asm(COPY, left4,  src, 16); \\\n\
        right4    -= left4; \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        _viv_asm(COPY, tmp, dst4, 16); \\\n\
        dst.s0123 = tmp.s1357; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
    _viv_asm(COPY, right4, src, 16); \\\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
    _viv_asm(COPY, left4,  src, 16); \\\n\
    right4    -= left4; \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
    _viv_asm(COPY, right4, src, 16); \\\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
    _viv_asm(COPY, left4,  src, 16); \\\n\
    right4    -= left4; \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    _viv_asm(COPY, tmp, dst4, 16); \\\n\
    dst.s0123 = tmp.s1357; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_BF16_BF16toBF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    vxc_short8 src;\n\
    VXC_DP2x8(src, read_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniBF16toFp32_part0_2x8);\n\
    _viv_asm(COPY, fxy0, src, 16);\n\
    VXC_DP2x8(src, read_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniBF16toFp32_part1_2x8);\n\
    _viv_asm(COPY, fxy1, src, 16);\n\
\n\
\n\
\n\
    GRID_SAMPLE_BF16_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_BF16_to_BF16_vx*/

static const char bilinear_grid_sample_F16_to_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniEvenBintoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniOddSubEvenBin_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
\n\
#define GRID_SAMPLE_F16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 t0; \\\n\
    vxc_short8 b0; \\\n\
    vxc_half8 top; \\\n\
    vxc_half8 bottom; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, top, t0, 16); \\\n\
    _viv_asm(COPY, bottom, b0, 16); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        half4 tmp; \\\n\
        _viv_asm(CONV, tmp, dst4); \\\n\
        VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8); \\\n\
        vxc_short4 result; \\\n\
        _viv_asm(COPY, result, top, 8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, top, t0, 16); \\\n\
        _viv_asm(COPY, bottom, b0, 16); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    half4 tmp; \\\n\
    _viv_asm(CONV, tmp, dst4); \\\n\
    VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8); \\\n\
    vxc_short4 result; \\\n\
    _viv_asm(COPY, result, top, 8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_F16_F32toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_F16_U8toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_uchar16 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_F16_F16toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_F16_to_F16_vx*/

static const char bilinear_grid_sample_F16_to_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniEvenBintoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniOddSubEvenBin_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
\n\
#define GRID_SAMPLE_F16_to_U8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 t0; \\\n\
    vxc_short8 b0; \\\n\
    vxc_uchar16 result; \\\n\
    vxc_half8 top; \\\n\
    vxc_half8 bottom; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, top, t0, 16); \\\n\
    _viv_asm(COPY, bottom, b0, 16); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * uint8Scale + output_ZP; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
        result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, top, t0, 16); \\\n\
        _viv_asm(COPY, bottom, b0, 16); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * uint8Scale + output_ZP; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
    result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void bilinear_grid_sample_F16_F32toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
\n\
__kernel void bilinear_grid_sample_F16_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_uchar16 read_coord;\n\
\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_F16_F16toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
"; /* end of bilinear_grid_sample_F16_to_U8_vx*/

static const char bilinear_grid_sample_I16_to_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part1_4x4;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform float dfpScale;\n\
\n\
#define GRID_SAMPLE_I16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 top; \\\n\
    vxc_short8 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * dfpScale; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * dfpScale; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_I16_I16toI16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_short8 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1_scale;\n\
    fxy1 = fxy1 * input1_scale;\n\
\n\
    GRID_SAMPLE_I16_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_I16_to_I16_vx*/

static const char bilinear_grid_sample_I8_to_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part1_4x4;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform float dfpScale;\n\
\n\
#define GRID_SAMPLE_I8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_char16 top; \\\n\
    vxc_char16 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * dfpScale; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * dfpScale; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_I8_I8toI8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_char16 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1_scale;\n\
    fxy1 = fxy1 * input1_scale;\n\
\n\
    GRID_SAMPLE_I8_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_I8_to_I8_vx*/

static const char bilinear_grid_sample_U8_to_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
\n\
#define GRID_SAMPLE_U8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_uchar16 top; \\\n\
    vxc_uchar16 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        unsigned char inputZP; \\\n\
        _viv_asm(COPY, inputZP, input_ZP, 4); \\\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, inputZP, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * uint8Scale + output_ZP; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    unsigned char inputZP; \\\n\
    _viv_asm(COPY, inputZP, input_ZP, 4); \\\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * uint8Scale + output_ZP; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void bilinear_grid_sample_U8_F32toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
\n\
__kernel void bilinear_grid_sample_U8_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_uchar16 read_coord;\n\
\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_U8_F16toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
"; /* end of bilinear_grid_sample_U8_to_U8_vx*/

static const char bilinear_grid_sample_reflect_BF16_to_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_even_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_odd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniBF16toFp32_part0_2x8;\n\
_viv_uniform VXC_512Bits uniBF16toFp32_part1_2x8;\n\
_viv_uniform float4 span_wh;\n\
_viv_uniform float4 min_val_wh;\n\
\n\
#define GRID_SAMPLE_BF16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy0 = fabs(fxy0 - min_val_wh); \\\n\
    fxy1 = fabs(fxy1 - min_val_wh); \\\n\
    float4 flips_xy0 = floor(fxy0 / span_wh); \\\n\
    float4 flips_xy1 = floor(fxy1 / span_wh); \\\n\
    float4 extra_xy0 = fxy0 - flips_xy0 * span_wh; \\\n\
    float4 extra_xy1 = fxy1 - flips_xy1 * span_wh; \\\n\
    int4 flips_int_xy0 = convert_int4(flips_xy0); \\\n\
    int4 flips_int_xy1 = convert_int4(flips_xy1); \\\n\
    fxy0 = ((flips_int_xy0 % 2) == 0) ? min_val_wh + extra_xy0 : min_val_wh + (span_wh - extra_xy0); \\\n\
    fxy1 = ((flips_int_xy1 % 2) == 0) ? min_val_wh + extra_xy1 : min_val_wh + (span_wh - extra_xy1); \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 top; \\\n\
    vxc_short8 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    vxc_ushort8 tmp, dst; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
        _viv_asm(COPY, right4, src, 16); \\\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
        _viv_asm(COPY, left4,  src, 16); \\\n\
        right4    -= left4; \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
        _viv_asm(COPY, right4, src, 16); \\\n\
        VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
        _viv_asm(COPY, left4,  src, 16); \\\n\
        right4    -= left4; \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        _viv_asm(COPY, tmp, dst4, 16); \\\n\
        dst.s0123 = tmp.s1357; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
    _viv_asm(COPY, right4, src, 16); \\\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
    _viv_asm(COPY, left4,  src, 16); \\\n\
    right4    -= left4; \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8); \\\n\
    _viv_asm(COPY, right4, src, 16); \\\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8); \\\n\
    _viv_asm(COPY, left4,  src, 16); \\\n\
    right4    -= left4; \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    _viv_asm(COPY, tmp, dst4, 16); \\\n\
    dst.s0123 = tmp.s1357; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_BF16_BF16toBF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    vxc_short8 src;\n\
    VXC_DP2x8(src, read_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniBF16toFp32_part0_2x8);\n\
    _viv_asm(COPY, fxy0, src, 16);\n\
    VXC_DP2x8(src, read_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniBF16toFp32_part1_2x8);\n\
    _viv_asm(COPY, fxy1, src, 16);\n\
\n\
\n\
\n\
    GRID_SAMPLE_BF16_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_reflect_BF16_to_BF16_vx*/

static const char bilinear_grid_sample_reflect_F16_to_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniEvenBintoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniOddSubEvenBin_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
_viv_uniform float4 span_wh;\n\
_viv_uniform float4 min_val_wh;\n\
\n\
#define GRID_SAMPLE_F16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy0 = fabs(fxy0 - min_val_wh); \\\n\
    fxy1 = fabs(fxy1 - min_val_wh); \\\n\
    float4 flips_xy0 = floor(fxy0 / span_wh); \\\n\
    float4 flips_xy1 = floor(fxy1 / span_wh); \\\n\
    float4 extra_xy0 = fxy0 - flips_xy0 * span_wh; \\\n\
    float4 extra_xy1 = fxy1 - flips_xy1 * span_wh; \\\n\
    int4 flips_int_xy0 = convert_int4(flips_xy0); \\\n\
    int4 flips_int_xy1 = convert_int4(flips_xy1); \\\n\
    fxy0 = ((flips_int_xy0 % 2) == 0) ? min_val_wh + extra_xy0 : min_val_wh + (span_wh - extra_xy0); \\\n\
    fxy1 = ((flips_int_xy1 % 2) == 0) ? min_val_wh + extra_xy1 : min_val_wh + (span_wh - extra_xy1); \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 t0; \\\n\
    vxc_short8 b0; \\\n\
    vxc_half8 top; \\\n\
    vxc_half8 bottom; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, top, t0, 16); \\\n\
    _viv_asm(COPY, bottom, b0, 16); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        half4 tmp; \\\n\
        _viv_asm(CONV, tmp, dst4); \\\n\
        VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8); \\\n\
        vxc_short4 result; \\\n\
        _viv_asm(COPY, result, top, 8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, top, t0, 16); \\\n\
        _viv_asm(COPY, bottom, b0, 16); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    half4 tmp; \\\n\
    _viv_asm(CONV, tmp, dst4); \\\n\
    VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8); \\\n\
    vxc_short4 result; \\\n\
    _viv_asm(COPY, result, top, 8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_F16_F32toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_reflect_F16_U8toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_uchar16 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_reflect_F16_F16toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_reflect_F16_to_F16_vx*/

static const char bilinear_grid_sample_reflect_F16_to_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniEvenBintoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniOddSubEvenBin_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float4 span_wh;\n\
_viv_uniform float4 min_val_wh;\n\
\n\
#define GRID_SAMPLE_F16_to_U8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy0 = fabs(fxy0 - min_val_wh); \\\n\
    fxy1 = fabs(fxy1 - min_val_wh); \\\n\
    float4 flips_xy0 = floor(fxy0 / span_wh); \\\n\
    float4 flips_xy1 = floor(fxy1 / span_wh); \\\n\
    float4 extra_xy0 = fxy0 - flips_xy0 * span_wh; \\\n\
    float4 extra_xy1 = fxy1 - flips_xy1 * span_wh; \\\n\
    int4 flips_int_xy0 = convert_int4(flips_xy0); \\\n\
    int4 flips_int_xy1 = convert_int4(flips_xy1); \\\n\
    fxy0 = ((flips_int_xy0 % 2) == 0) ? min_val_wh + extra_xy0 : min_val_wh + (span_wh - extra_xy0); \\\n\
    fxy1 = ((flips_int_xy1 % 2) == 0) ? min_val_wh + extra_xy1 : min_val_wh + (span_wh - extra_xy1); \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 t0; \\\n\
    vxc_short8 b0; \\\n\
    vxc_uchar16 result; \\\n\
    vxc_half8 top; \\\n\
    vxc_half8 bottom; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, top, t0, 16); \\\n\
    _viv_asm(COPY, bottom, b0, 16); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * uint8Scale + output_ZP; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
        result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, t0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, b0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, top, t0, 16); \\\n\
        _viv_asm(COPY, bottom, b0, 16); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniEvenBintoFp32_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniOddSubEvenBin_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * uint8Scale + output_ZP; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
    result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_F16_F32toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_F16_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_uchar16 read_coord;\n\
\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_reflect_F16_F16toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
"; /* end of bilinear_grid_sample_reflect_F16_to_U8_vx*/

static const char bilinear_grid_sample_reflect_I16_to_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part1_4x4;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float4 span_wh;\n\
_viv_uniform float4 min_val_wh;\n\
\n\
#define GRID_SAMPLE_I16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy0 = fabs(fxy0 - min_val_wh); \\\n\
    fxy1 = fabs(fxy1 - min_val_wh); \\\n\
    float4 flips_xy0 = floor(fxy0 / span_wh); \\\n\
    float4 flips_xy1 = floor(fxy1 / span_wh); \\\n\
    float4 extra_xy0 = fxy0 - flips_xy0 * span_wh; \\\n\
    float4 extra_xy1 = fxy1 - flips_xy1 * span_wh; \\\n\
    int4 flips_int_xy0 = convert_int4(flips_xy0); \\\n\
    int4 flips_int_xy1 = convert_int4(flips_xy1); \\\n\
    fxy0 = ((flips_int_xy0 % 2) == 0) ? min_val_wh + extra_xy0 : min_val_wh + (span_wh - extra_xy0); \\\n\
    fxy1 = ((flips_int_xy1 % 2) == 0) ? min_val_wh + extra_xy1 : min_val_wh + (span_wh - extra_xy1); \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 top; \\\n\
    vxc_short8 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * dfpScale; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * dfpScale; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_I16_I16toI16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_short8 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1_scale;\n\
    fxy1 = fxy1 * input1_scale;\n\
\n\
    GRID_SAMPLE_I16_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_reflect_I16_to_I16_vx*/

static const char bilinear_grid_sample_reflect_I8_to_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part1_4x4;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float4 span_wh;\n\
_viv_uniform float4 min_val_wh;\n\
\n\
#define GRID_SAMPLE_I8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy0 = fabs(fxy0 - min_val_wh); \\\n\
    fxy1 = fabs(fxy1 - min_val_wh); \\\n\
    float4 flips_xy0 = floor(fxy0 / span_wh); \\\n\
    float4 flips_xy1 = floor(fxy1 / span_wh); \\\n\
    float4 extra_xy0 = fxy0 - flips_xy0 * span_wh; \\\n\
    float4 extra_xy1 = fxy1 - flips_xy1 * span_wh; \\\n\
    int4 flips_int_xy0 = convert_int4(flips_xy0); \\\n\
    int4 flips_int_xy1 = convert_int4(flips_xy1); \\\n\
    fxy0 = ((flips_int_xy0 % 2) == 0) ? min_val_wh + extra_xy0 : min_val_wh + (span_wh - extra_xy0); \\\n\
    fxy1 = ((flips_int_xy1 % 2) == 0) ? min_val_wh + extra_xy1 : min_val_wh + (span_wh - extra_xy1); \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_char16 top; \\\n\
    vxc_char16 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * dfpScale; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * dfpScale; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_I8_I8toI8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_char16 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1_scale;\n\
    fxy1 = fxy1 * input1_scale;\n\
\n\
    GRID_SAMPLE_I8_PROCESS();\n\
\n\
}\n\
"; /* end of bilinear_grid_sample_reflect_I8_to_I8_vx*/

static const char bilinear_grid_sample_reflect_U8_to_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
_viv_uniform float4 span_wh;\n\
_viv_uniform float4 min_val_wh;\n\
\n\
#define GRID_SAMPLE_U8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy0 = fabs(fxy0 - min_val_wh); \\\n\
    fxy1 = fabs(fxy1 - min_val_wh); \\\n\
    float4 flips_xy0 = floor(fxy0 / span_wh); \\\n\
    float4 flips_xy1 = floor(fxy1 / span_wh); \\\n\
    float4 extra_xy0 = fxy0 - flips_xy0 * span_wh; \\\n\
    float4 extra_xy1 = fxy1 - flips_xy1 * span_wh; \\\n\
    int4 flips_int_xy0 = convert_int4(flips_xy0); \\\n\
    int4 flips_int_xy1 = convert_int4(flips_xy1); \\\n\
    fxy0 = ((flips_int_xy0 % 2) == 0) ? min_val_wh + extra_xy0 : min_val_wh + (span_wh - extra_xy0); \\\n\
    fxy1 = ((flips_int_xy1 % 2) == 0) ? min_val_wh + extra_xy1 : min_val_wh + (span_wh - extra_xy1); \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    float4 x_f         = floor(in_x); \\\n\
    float4 x_lerp      = in_x - x_f; \\\n\
    int4   x_idx       = convert_int4(x_f); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    float4  y_f         = floor(in_y); \\\n\
    float4  y_lerp       = in_y - y_f; \\\n\
    int4    y_idx        = convert_int4(y_f); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_uchar16 top; \\\n\
    vxc_uchar16 bottom; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 left4; \\\n\
    float4 right4; \\\n\
    float4 top4; \\\n\
    float4 bottom4; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        unsigned char inputZP; \\\n\
        _viv_asm(COPY, inputZP, input_ZP, 4); \\\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
        top4        = right4 * x_lerp + left4; \\\n\
        VXC_DP4x4(left4, bottom, inputZP, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
        bottom4      = right4 * x_lerp + left4; \\\n\
        bottom4     -= top4; \\\n\
        float4 dst4  = bottom4 * y_lerp + top4; \\\n\
        dst4         = dst4 * uint8Scale + output_ZP; \\\n\
        int4 dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, top, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, bottom, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    unsigned char inputZP; \\\n\
    _viv_asm(COPY, inputZP, input_ZP, 4); \\\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * uint8Scale + output_ZP; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_U8_F32toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_U8_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_uchar16 read_coord;\n\
\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void bilinear_grid_sample_reflect_U8_F16toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
"; /* end of bilinear_grid_sample_reflect_U8_to_U8_vx*/

static const char bucketize_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataConvert_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataConvert_1_4x4;\n\
_viv_uniform int boundaries_size_x8;\n\
_viv_uniform int boundaries_size;\n\
\n\
#define BUCKETIZE_16BITS_SH_IMPL(name, copy_type) \\\n\
__kernel void bucketize_right_##name \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t boundaries, \\\n\
    __write_only image2d_t output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    vxc_short8 data0, data1; \\\n\
    copy_type src0, src1, dst0, dst1; \\\n\
    vxc_ushort8 v0, v1, v2, v3, result = 0; \\\n\
    VXC_ReadImage(data0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
 \\\n\
    for (; coord.z < boundaries_size_x8; ) \\\n\
    { \\\n\
        VXC_ReadImage(data1, boundaries, coord.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src1, data1.s00000000, 16); \\\n\
        coord.z += 8; \\\n\
 \\\n\
        VXC_Clamp(dst0, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v0, dst0, 16); \\\n\
        v2 = sub_sat(v0, 0xFFFE); \\\n\
        _viv_asm(COPY, src1, data1.s11111111, 16); \\\n\
        VXC_Clamp(dst1, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v1, dst1, 16); \\\n\
        v3 = sub_sat(v1, 0xFFFE); \\\n\
 \\\n\
        result = result + v2 + v3; \\\n\
 \\\n\
        _viv_asm(COPY, src1, data1.s22222222, 16); \\\n\
        VXC_Clamp(dst0, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v0, dst0, 16); \\\n\
        v2 = sub_sat(v0, 0xFFFE); \\\n\
        _viv_asm(COPY, src1, data1.s33333333, 16); \\\n\
        VXC_Clamp(dst1, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v1, dst1, 16); \\\n\
        v3 = sub_sat(v1, 0xFFFE); \\\n\
 \\\n\
        result = result + v2 + v3; \\\n\
 \\\n\
        _viv_asm(COPY, src1, data1.s44444444, 16); \\\n\
        VXC_Clamp(dst0, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v0, dst0, 16); \\\n\
        v2 = sub_sat(v0, 0xFFFE); \\\n\
        _viv_asm(COPY, src1, data1.s55555555, 16); \\\n\
        VXC_Clamp(dst1, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v1, dst1, 16); \\\n\
        v3 = sub_sat(v1, 0xFFFE); \\\n\
 \\\n\
        result = result + v2 + v3; \\\n\
 \\\n\
        _viv_asm(COPY, src1, data1.s66666666, 16); \\\n\
        VXC_Clamp(dst0, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v0, dst0, 16); \\\n\
        v2 = sub_sat(v0, 0xFFFE); \\\n\
        _viv_asm(COPY, src1, data1.s77777777, 16); \\\n\
        VXC_Clamp(dst1, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v1, dst1, 16); \\\n\
        v3 = sub_sat(v1, 0xFFFE); \\\n\
 \\\n\
        result = result + v2 + v3; \\\n\
    } \\\n\
 \\\n\
    for (; coord.z < boundaries_size; ) \\\n\
    { \\\n\
        VXC_ReadImage(data1, boundaries, coord.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src1, data1.s00000000, 16); \\\n\
        coord.z ++; \\\n\
 \\\n\
        VXC_Clamp(dst0, src0, src1, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, v0, dst0, 16); \\\n\
        v2 = sub_sat(v0, 0xFFFE); \\\n\
 \\\n\
        result = result + v2; \\\n\
    } \\\n\
 \\\n\
    int4 d0, d1; \\\n\
    VXC_DP4x4(d0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniDataConvert_0_4x4); \\\n\
    VXC_DP4x4(d1, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniDataConvert_1_4x4); \\\n\
    coord.z = coord.x + 4; \\\n\
 \\\n\
    write_imagei(output, coord.xy, d0); \\\n\
    write_imagei(output, coord.zy, d1); \\\n\
}\n\
BUCKETIZE_16BITS_SH_IMPL(F16_F16toI32_2D, vxc_half8)\n\
BUCKETIZE_16BITS_SH_IMPL(I16_I16toI32_2D, vxc_short8)\n\
\n\
#define BUCKETIZE_8BITS_SH_IMPL(name, src_type) \\\n\
__kernel void bucketize_right_##name \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t boundaries, \\\n\
    __write_only image2d_t output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    src_type src0, src1, src2; \\\n\
    vxc_uchar8 dst0, dst1, result = 0; \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, boundaries, coord.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    for (; coord.z < boundaries_size_x8; ) \\\n\
    { \\\n\
        VXC_ReadImage(src1, boundaries, coord.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z += 8; \\\n\
 \\\n\
        VXC_Clamp(src2, src0, src1.s00000000, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst0, src2, 8); \\\n\
        dst0 = sub_sat(dst0, 0xFE); \\\n\
        VXC_Clamp(src2, src0, src1.s11111111, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst1, src2, 8); \\\n\
        dst1 = sub_sat(dst1, 0xFE); \\\n\
 \\\n\
        result = result + dst0 + dst1; \\\n\
 \\\n\
        VXC_Clamp(src2, src0, src1.s22222222, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst0, src2, 8); \\\n\
        dst0 = sub_sat(dst0, 0xFE); \\\n\
        VXC_Clamp(src2, src0, src1.s33333333, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst1, src2, 8); \\\n\
        dst1 = sub_sat(dst1, 0xFE); \\\n\
 \\\n\
        result = result + dst0 + dst1; \\\n\
 \\\n\
        VXC_Clamp(src2, src0, src1.s44444444, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst0, src2, 8); \\\n\
        dst0 = sub_sat(dst0, 0xFE); \\\n\
        VXC_Clamp(src2, src0, src1.s55555555, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst1, src2, 8); \\\n\
        dst1 = sub_sat(dst1, 0xFE); \\\n\
 \\\n\
        result = result + dst0 + dst1; \\\n\
 \\\n\
        VXC_Clamp(src2, src0, src1.s66666666, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst0, src2, 8); \\\n\
        dst0 = sub_sat(dst0, 0xFE); \\\n\
        VXC_Clamp(src2, src0, src1.s77777777, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst1, src2, 8); \\\n\
        dst1 = sub_sat(dst1, 0xFE); \\\n\
 \\\n\
        result = result + dst0 + dst1; \\\n\
    } \\\n\
 \\\n\
    for (; coord.z < boundaries_size; ) \\\n\
    { \\\n\
        VXC_ReadImage(src1, boundaries, coord.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z ++; \\\n\
 \\\n\
        VXC_Clamp(src2, src0, src1.s00000000, src0, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
        _viv_asm(COPY, dst0, src2, 8); \\\n\
        dst0 = sub_sat(dst0, 0xFE); \\\n\
 \\\n\
        result = result + dst0; \\\n\
    } \\\n\
 \\\n\
    int4 d0, d1; \\\n\
    VXC_DP4x4(d0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniDataConvert_0_4x4); \\\n\
    VXC_DP4x4(d1, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniDataConvert_1_4x4); \\\n\
    coord.z = coord.x + 4; \\\n\
 \\\n\
    write_imagei(output, coord.xy, d0); \\\n\
    write_imagei(output, coord.zy, d1); \\\n\
}\n\
BUCKETIZE_8BITS_SH_IMPL(U8_U8toI32_2D, vxc_uchar8)\n\
BUCKETIZE_8BITS_SH_IMPL(I8_I8toI32_2D, vxc_char8)\n\
"; /* end of bucketize_vx*/

static const char cast_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataConvert_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
#define CAST_PROCESS(read_type, src_type, dst_type, write_type, read_fun, write_fun) \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst_val; \\\n\
    write_type write_val; \\\n\
    read_fun(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    VXC_DP2x8(dst_val, src_val, src_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniDataConvert_2x8); \\\n\
    _viv_asm(COPY, write_val, dst_val, 16); \\\n\
    write_fun(output, coord, write_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define CAST_FUN(src_name, dst_name, read_type, src_type, dst_type, write_type) \\\n\
__kernel void cast_##src_name##to##dst_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    CAST_PROCESS(read_type, src_type, dst_type, write_type, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
CAST_FUN(F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
CAST_FUN(F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8)\n\
CAST_FUN(F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN(I16, F16, vxc_short8, vxc_short8, vxc_half8,  vxc_short8)\n\
CAST_FUN(I16, I8,  vxc_short8, vxc_short8, vxc_char8,  vxc_char8)\n\
CAST_FUN(I16, U8,  vxc_short8, vxc_short8, vxc_uchar8, vxc_uchar8)\n\
CAST_FUN(I8, F16, vxc_char8,   vxc_char8,  vxc_half8,  vxc_short8)\n\
CAST_FUN(I8, I16, vxc_char8,   vxc_char8,  vxc_short8, vxc_short8)\n\
CAST_FUN(I8, U8,  vxc_char8,   vxc_char8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN(U8, F16, vxc_uchar8,  vxc_uchar8, vxc_half8,  vxc_short8)\n\
CAST_FUN(U8, I16, vxc_uchar8,  vxc_uchar8, vxc_short8, vxc_short8)\n\
CAST_FUN(U8, I8,  vxc_uchar8,  vxc_uchar8, vxc_char8,  vxc_char8)\n\
\n\
\n\
#define CAST_FUN_2D(src_name, dst_name, read_type, src_type, dst_type, write_type) \\\n\
__kernel void cast_##src_name##to##dst_name##_2D( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    CAST_PROCESS(read_type, src_type, dst_type, write_type, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
CAST_FUN_2D(F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
CAST_FUN_2D(F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8)\n\
CAST_FUN_2D(F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN_2D(I16, F16, vxc_short8, vxc_short8, vxc_half8,  vxc_short8)\n\
CAST_FUN_2D(I16, I8,  vxc_short8, vxc_short8, vxc_char8,  vxc_char8)\n\
CAST_FUN_2D(I16, U8,  vxc_short8, vxc_short8, vxc_uchar8, vxc_uchar8)\n\
CAST_FUN_2D(I8, F16, vxc_char8,   vxc_char8,  vxc_half8,  vxc_short8)\n\
CAST_FUN_2D(I8, I16, vxc_char8,   vxc_char8,  vxc_short8, vxc_short8)\n\
CAST_FUN_2D(I8, U8,  vxc_char8,   vxc_char8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN_2D(U8, F16, vxc_uchar8,  vxc_uchar8, vxc_half8,  vxc_short8)\n\
CAST_FUN_2D(U8, I16, vxc_uchar8,  vxc_uchar8, vxc_short8, vxc_short8)\n\
CAST_FUN_2D(U8, I8,  vxc_uchar8,  vxc_uchar8, vxc_char8,  vxc_char8)\n\
\n\
#define CAST_TO_BOOL_PROCESS(src_type, tmp_type, read_fun, write_fun) \\\n\
    src_type   src_val; \\\n\
    tmp_type   tmp_val; \\\n\
    vxc_char8   dst_val; \\\n\
    read_fun(src_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    tmp_val  = (src_val != 0); \\\n\
    tmp_val *= (-1); \\\n\
    VXC_DP2x8(dst_val, tmp_val, tmp_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDataConvert_2x8); \\\n\
    write_fun(output, coord, dst_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define CAST_TO_BOOL_FUN(src_name, src_type, tmp_type) \\\n\
__kernel void cast_##src_name##toBOOL8( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    CAST_TO_BOOL_PROCESS(src_type, tmp_type, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN(F16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN(I16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN(I8,  vxc_char8,  vxc_char8)\n\
CAST_TO_BOOL_FUN(U8,  vxc_uchar8, vxc_char8)\n\
\n\
#define CAST_TO_BOOL_FUN_2D(src_name, src_type, tmp_type) \\\n\
__kernel void cast_##src_name##toBOOL8_2D( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    CAST_TO_BOOL_PROCESS(src_type, tmp_type, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN_2D(F16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN_2D(I16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN_2D(I8,  vxc_char8,  vxc_char8)\n\
CAST_TO_BOOL_FUN_2D(U8,  vxc_uchar8, vxc_char8)\n\
\n\
#define CAST_F32orI32_PROCESS(src_type, dst_type, read_fun, write_fun) \\\n\
    src_type   src_val0, src_val1; \\\n\
    dst_type   dst_val; \\\n\
    int4 tmpData1, tmpData2; \\\n\
    src_val0 = read_fun(input, coord); \\\n\
    coord.x += 4; \\\n\
    src_val1 = read_fun(input, coord); \\\n\
    tmpData1 = convert_int4(src_val0); \\\n\
    tmpData2 = convert_int4(src_val1); \\\n\
    VXC_DP2x8(dst_val, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    coord.x -= 4; \\\n\
    write_fun(output, coord, dst_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define CAST_F32orI32_FUN(src_name, dst_name, src_type, dst_type, read_fun) \\\n\
__kernel void cast_##src_name##to##dst_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    CAST_F32orI32_PROCESS(src_type, dst_type, read_fun, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
CAST_F32orI32_FUN(F32, I16, float4, vxc_short8, read_imagef)\n\
CAST_F32orI32_FUN(F32, I8,  float4, vxc_char8,  read_imagef)\n\
CAST_F32orI32_FUN(F32, U8,  float4, vxc_uchar8, read_imagef)\n\
CAST_F32orI32_FUN(I32, I16, int4,   vxc_short8, read_imagei)\n\
CAST_F32orI32_FUN(I32, I8,  int4,   vxc_char8,  read_imagei)\n\
CAST_F32orI32_FUN(I32, U8,  int4,   vxc_uchar8, read_imagei)\n\
\n\
\n\
#define CAST_F32orI32_FUN_2D(src_name, dst_name, src_type, dst_type, read_fun) \\\n\
__kernel void cast_##src_name##to##dst_name##_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    CAST_F32orI32_PROCESS(src_type, dst_type, read_fun, VXC_WriteImage) \\\n\
}\n\
\n\
CAST_F32orI32_FUN_2D(F32, I16, float4, vxc_short8, read_imagef)\n\
CAST_F32orI32_FUN_2D(F32, I8,  float4, vxc_char8,  read_imagef)\n\
CAST_F32orI32_FUN_2D(F32, U8,  float4, vxc_uchar8, read_imagef)\n\
CAST_F32orI32_FUN_2D(I32, I16, int4,   vxc_short8, read_imagei)\n\
CAST_F32orI32_FUN_2D(I32, I8,  int4,   vxc_char8,  read_imagei)\n\
CAST_F32orI32_FUN_2D(I32, U8,  int4,   vxc_uchar8, read_imagei)\n\
\n\
"; /* end of cast_vx*/

static const char clip_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedMinData_FP16;\n\
_viv_uniform int4 packedMaxData_FP16;\n\
_viv_uniform VXC_512Bits uniConvertF16toInt_2x8;\n\
_viv_uniform int2 multAndoutZP;\n\
_viv_uniform VXC_512Bits uniDataMulAndPostShift_2x8;\n\
\n\
#define TENSORCLIP_F16TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 vec0, dst; \\\n\
    vxc_half8  src0, src1, minHf, maxHf; \\\n\
    read_fun(vec0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vec0, 16); \\\n\
    _viv_asm(COPY, minHf, packedMinData_FP16, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData_FP16, 16); \\\n\
    VXC_Clamp_Half(src1, src0, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src1, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_F16toF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_F16toF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_F16TOINT_PROCESS(read_fun, write_fun, dst_type) \\\n\
    vxc_short8 vec0; \\\n\
    dst_type dst; \\\n\
    vxc_half8  src0, src1, minHf, maxHf; \\\n\
    read_fun(vec0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vec0, 16); \\\n\
    _viv_asm(COPY, minHf, packedMinData_FP16, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData_FP16, 16); \\\n\
    VXC_Clamp_Half(src1, src0, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    VXC_DP2x8(dst, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertF16toInt_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_F16toI16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, vxc_short8)\n\
}\n\
\n\
__kernel void clip_F16toI16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage, VXC_WriteImage, vxc_short8)\n\
}\n\
\n\
__kernel void clip_F16toI8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, vxc_char16)\n\
}\n\
\n\
__kernel void clip_F16toI8_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage, VXC_WriteImage, vxc_char16)\n\
}\n\
\n\
#define TENSORCLIP_F16TOU8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 vec0; \\\n\
    vxc_uchar16 dst; \\\n\
    vxc_half8  src0, src1, minHf, maxHf; \\\n\
    read_fun(vec0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vec0, 16); \\\n\
    _viv_asm(COPY, minHf, packedMinData_FP16, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData_FP16, 16); \\\n\
    VXC_Clamp_Half(src1, src0, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src1, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniDataMulAndPostShift_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_F16toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOU8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_F16toU8_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOU8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of clip_F16_vx*/

static const char clip_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntegerLo_2x8;\n\
_viv_uniform int4 packedMinData;\n\
_viv_uniform int4 packedMaxData;\n\
\n\
#define TENSORCLIP_I16TOI16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, min, max; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    _viv_asm(COPY, min, packedMinData, 16); \\\n\
    _viv_asm(COPY, max, packedMaxData, 16); \\\n\
    VXC_Clamp(src0, src0, min, max, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    write_fun(output, coord, src0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_I16toI16(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I16TOI16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I16toI16_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I16TOI16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_I16TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, dst; \\\n\
    vxc_half8  src1, src2, minHf, maxHf; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src1, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    _viv_asm(COPY, minHf, packedMinData, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData, 16); \\\n\
    VXC_Clamp_Half(src2, src1, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src2, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void clip_I16toF16(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I16TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I16toF16_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I16TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of clip_I16_vx*/

static const char clip_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntegerLo_2x8;\n\
_viv_uniform VXC_512Bits uniConvertIntegerHi_2x8;\n\
_viv_uniform int4 packedMinData;\n\
_viv_uniform int4 packedMaxData;\n\
\n\
#define TENSORCLIP_I8TOI8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 src0, min, max; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerHi_2x8); \\\n\
    _viv_asm(COPY, min, packedMinData, 16); \\\n\
    _viv_asm(COPY, max, packedMaxData, 16); \\\n\
    VXC_Clamp(src0, src0, min, max, VXC_MODIFIER_CLAMP(0, 15, 0, 0)); \\\n\
    write_fun(output, coord, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_I8toI8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I8TOI8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I8toI8_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I8TOI8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_I8TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 src0; \\\n\
    vxc_short8 dst0, dst1; \\\n\
    vxc_half8  src1, src2, src3, src4, minHf, maxHf; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src1, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    VXC_DP2x8(src2, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerHi_2x8); \\\n\
    _viv_asm(COPY, minHf, packedMinData, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData, 16); \\\n\
    VXC_Clamp_Half(src3, src1, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    VXC_Clamp_Half(src4, src2, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst0, src3, 16); \\\n\
    _viv_asm(COPY, dst1, src4, 16); \\\n\
    write_fun(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    write_fun(output, coord, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_I8toF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I8TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I8toF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I8TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of clip_I8_vx*/

static const char clip_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedMinData;\n\
_viv_uniform int4 packedMaxData;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define TENSORCLIP_U8TOU8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 vec0, min, max, dst; \\\n\
    read_fun(vec0, input,  coord,\\\n\
         VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, vec0, multiplier,\\\n\
         VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(dst, vec0, multiplier,\\\n\
         VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Hi_2x8); \\\n\
    _viv_asm(COPY, min, packedMinData, 16); \\\n\
    _viv_asm(COPY, max, packedMaxData, 16); \\\n\
    VXC_Clamp(dst, dst, min, max, VXC_MODIFIER_CLAMP(0, 15, 0, 0)); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void clip_U8toU8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_U8TOU8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_U8toU8_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_U8TOU8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_U8TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 vec0; \\\n\
    vxc_short8 dst0, dst1; \\\n\
    vxc_half8  src1, src2, src3, src4, minHf, maxHf; \\\n\
    read_fun(vec0, input,  coord,\\\n\
         VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(src1, vec0, multiplier,\\\n\
         VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(src2, vec0, multiplier,\\\n\
         VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Hi_2x8); \\\n\
    _viv_asm(COPY, minHf, packedMinData, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData, 16); \\\n\
    VXC_Clamp_Half(src3, src1, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    VXC_Clamp_Half(src4, src2, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst0, src3, 16); \\\n\
    _viv_asm(COPY, dst1, src4, 16); \\\n\
    write_fun(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    write_fun(output, coord, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void clip_U8toF16(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_U8TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_U8toF16_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_U8TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of clip_U8_vx*/

static const char conv1d_ovxlib_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConv1DK3_Lo0_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Lo1_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Lo2_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Hi0_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Hi1_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Hi2_4x4;\n\
_viv_uniform VXC_512Bits uniDataConvK3_2x8;\n\
_viv_uniform VXC_512Bits uniSumOrderUchar_2x8;\n\
\n\
_viv_uniform int input_ZP;\n\
_viv_uniform int weight_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float scaleOut;\n\
_viv_uniform int input_height;\n\
\n\
__kernel void conv1d_U8U8I32toU8_K3_S1(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int4 coord   = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 weight_val_s =(short)input_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
    weight_val_s.s5 = (short)weight_ZP;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(weight_val_s, weight_val, weight_val_s, \\\n\
                             VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0), uniDataConvK3_2x8);\n\
\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo0_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi0_4x4);\n\
        sum1 += dst;\n\
        coord.x += dilation;\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo1_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi1_4x4);\n\
        sum1 += dst;\n\
        coord.x += dilation;\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo2_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi2_4x4);\n\
        sum1 += dst;\n\
        coord_w.y++;\n\
        coord.z++;\n\
        coord.x = get_global_id(0);\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    VXC_WriteImage(output, coord.xy, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void conv1d_U8U8I32toU8_K3_S1_D2_D4(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int4 coord   = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 weight_val_s =(short)input_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
    weight_val_s.s5 = (short)weight_ZP;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(weight_val_s, weight_val, weight_val_s, \\\n\
                             VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0), uniDataConvK3_2x8);\n\
\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo0_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi0_4x4);\n\
        sum1 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo1_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi1_4x4);\n\
        sum1 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo2_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi2_4x4);\n\
        sum1 += dst;\n\
        coord_w.y++;\n\
        coord.z++;\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    VXC_WriteImage(output, coord.xy, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of conv1d_ovxlib_vx*/

static const char conv1d_ovxlib_k1024_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part0_8x2;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part1_8x2;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part2_8x2;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part3_8x2;\n\
_viv_uniform VXC_512Bits uniSumOrderUchar_2x8;\n\
\n\
_viv_uniform int kernel_cnt_x16;\n\
_viv_uniform int weight_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float scaleOut;\n\
_viv_uniform int input_height;\n\
_viv_uniform int input_width;\n\
_viv_uniform int output_width;\n\
\n\
__kernel void conv1d_U8U8I32toU8_K1024_SMALL(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int  start_x = get_global_id(0) - pad_front;\n\
    int4 coord   = (int4)(start_x, get_global_id(1), 0, get_global_id(0));\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 coef;\n\
    vxc_short8 w_zp = (short)weight_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i, j;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        for (j = 0; j < kernel_cnt_x16; j++)\n\
        {\n\
            VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(8, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            coord_w.x += 16;\n\
            coord.x += 16;\n\
        }\n\
        coord_w.x = 0;\n\
        coord_w.y++;\n\
        coord.z++;\n\
        coord.x = start_x;\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    VXC_WriteImage(output, coord.wy, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void conv1d_U8U8I32toU8_K1024_LARGE(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int  start_x = get_global_id(0);\n\
    int  w_left  = output_width - start_x;\n\
    int  out_x   = w_left < 8 ? get_global_id(0) - w_left : get_global_id(0);\n\
    int4 coord   = (int4)(start_x, get_global_id(1), 0, out_x);\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 coef;\n\
    vxc_short8 w_zp = (short)weight_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i, j;\n\
    Tensor src_tensor = create_tensor_from_image2d_array(input, 1);\n\
    uchar *src_ptr_base = (uchar *)src_tensor.ptr;\n\
    uchar *src_ptr;\n\
    Tensor dst_tensor = create_tensor_from_image2d_array(output, 1);\n\
    uchar *dst_ptr = (uchar *)dst_tensor.ptr;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        src_ptr = src_ptr_base + (coord.x + coord.z * src_tensor.stride_y);\n\
        for (j = 0; j < kernel_cnt_x16; j++)\n\
        {\n\
            VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_Vload16(input_val, src_ptr, 0);\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            src_ptr += 8;\n\
            VXC_Vload16(input_val, src_ptr, 0);\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            coord_w.x += 16;\n\
            coord.x += 16;\n\
            src_ptr += 8;\n\
        }\n\
        coord_w.x = 0;\n\
        coord_w.y++;\n\
        coord.z++;\n\
        coord.x = start_x;\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    dst_ptr = dst_ptr + (coord.w + coord.y * dst_tensor.stride_y);\n\
    VXC_Vstore8(dst_ptr, 0, result);\n\
}\n\
\n\
"; /* end of conv1d_ovxlib_k1024_vx*/

static const char crop_and_resize_bilinear_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float inOutScale;\n\
_viv_uniform float inOutTile;\n\
_viv_uniform float width_scale;\n\
_viv_uniform float height_scale;\n\
_viv_uniform int   image_width;\n\
_viv_uniform int   image_height;\n\
_viv_uniform VXC_512Bits uniRightToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniLeftToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
\n\
#define CROP_AND_RESIZE_PART0 \\\n\
    int i = 0; \\\n\
    int bb = get_global_id(2); \\\n\
    int y =  get_global_id(1); \\\n\
    int4 x = (int4)(get_global_id(0),get_global_id(0) + 1, get_global_id(0) + 2, get_global_id(0) + 3); \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int2 coord_box_ind = (int2)(bb, 0); \\\n\
    int b = read_imagei(box_ind, coord_box_ind).x; \\\n\
    float4 xy, in_x; \\\n\
    float in_y; \\\n\
    float4 x_lerp, y_lerp; \\\n\
    int d = 0; \\\n\
    Image img_boxes = create_image_from_image2d(boxes, 2); \\\n\
    __global half* boxes_ptr = (__global half*)img_boxes.ptr; \\\n\
    xy = vload_half4(bb, boxes_ptr); \\\n\
    float _width_scale = convert_float(xy.w - xy.y) * width_scale; \\\n\
    float _height_scale = convert_float(xy.z - xy.x) * height_scale; \\\n\
 \\\n\
    if (_width_scale == 0) xy.y = 0.5 * (xy.y + xy.w); \\\n\
    if (_height_scale == 0) xy.x = 0.5 * (xy.x + xy.z); \\\n\
    in_y = xy.x * convert_float(image_height - 1) + convert_float(y) * _height_scale; \\\n\
    y_lerp.x = in_y - floor(in_y); \\\n\
    y_lerp.yzw = y_lerp.xxx;\n\
\n\
#define CROP_AND_RESIZE_PART1 \\\n\
        int4 coord = (int4)(0, in_y, d + b * ori_depth, 0); \\\n\
        int8 input_desc, output_desc; \\\n\
 \\\n\
        coord_out.z = d + coord_out.z * ori_depth; \\\n\
 \\\n\
        _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
        int baseAddr = (int)coord.z * input_desc.s4 + input_desc.s0; \\\n\
        _viv_asm(MOV, coord.w, baseAddr); \\\n\
 \\\n\
        _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
        baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
        _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
        in_x.x = xy.y * convert_float(image_width - 1); \\\n\
        in_x.yzw = in_x.xxx; \\\n\
        in_x = in_x + convert_float4(x) * _width_scale; \\\n\
        x_lerp = in_x - floor(in_x); \\\n\
        coord.x = floor(in_x.x); \\\n\
        VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                     VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x = floor(in_x.y); \\\n\
        VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                     VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x = floor(in_x.z); \\\n\
        VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                     VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x = floor(in_x.w); \\\n\
        VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                     VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define CROP_AND_RESIZE_BILINEAR_Quant8toQuant8(name,src_type,dst_type) \\\n\
__kernel void crop_and_resize_bilinear_##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    CROP_AND_RESIZE_PART0; \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        src_type src0, src1; \\\n\
        CROP_AND_RESIZE_PART1; \\\n\
 \\\n\
        float4 top, bottom, value; \\\n\
        float4 top_left4,top_right4,bottom_left4,bottom_right4; \\\n\
        dst_type data; \\\n\
        int4 tmpout; \\\n\
 \\\n\
        VXC_DP4x4(top_left4, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(top_right4, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_left4, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_right4, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
 \\\n\
        top = top_left4 + (top_right4 - top_left4) * x_lerp; \\\n\
        bottom = bottom_left4 + (bottom_right4 - bottom_left4) * x_lerp; \\\n\
        value = top + (bottom - top) * y_lerp; \\\n\
        value = value * inOutScale + inOutTile; \\\n\
        _viv_asm(CONV, tmpout, value); \\\n\
        VXC_DP2x8(data, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtract8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_BILINEAR_Quant8toQuant8(U8toU8,vxc_uchar8, vxc_uchar4)\n\
CROP_AND_RESIZE_BILINEAR_Quant8toQuant8(I8toI8,vxc_char8, vxc_char4)\n\
\n\
#define CROP_AND_RESIZE_BILINEAR_16Bitsto16Bits(name,src_type,dst_type,tmp_type) \\\n\
__kernel void crop_and_resize_bilinear_##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    CROP_AND_RESIZE_PART0; \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        vxc_short8 src0, src1; \\\n\
        src_type src0_temp, src1_temp; \\\n\
        CROP_AND_RESIZE_PART1; \\\n\
 \\\n\
        _viv_asm(COPY, src0_temp, src0, 16); \\\n\
        _viv_asm(COPY, src1_temp, src1, 16); \\\n\
        float4 top, bottom, value; \\\n\
        float4 top_left4,top_right4,bottom_left4,bottom_right4; \\\n\
        dst_type data; \\\n\
        vxc_short4 out; \\\n\
        tmp_type tmpout; \\\n\
 \\\n\
        VXC_DP4x4(top_left4, src0_temp, src0_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(top_right4, src0_temp, src0_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_left4, src1_temp, src1_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_right4, src1_temp, src1_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
 \\\n\
        top = top_left4 + (top_right4 - top_left4) * x_lerp; \\\n\
        bottom = bottom_left4 + (bottom_right4 - bottom_left4) * x_lerp; \\\n\
        value = top + (bottom - top) * y_lerp; \\\n\
        value = value * inOutScale + inOutTile; \\\n\
        _viv_asm(CONV, tmpout, value); \\\n\
        VXC_DP2x8(data, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8); \\\n\
        _viv_asm(COPY, out, data, 8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_BILINEAR_16Bitsto16Bits(I16toI16, vxc_short8, vxc_short4, short4)\n\
CROP_AND_RESIZE_BILINEAR_16Bitsto16Bits(I16toF16, vxc_short8, vxc_half4,  half4)\n\
CROP_AND_RESIZE_BILINEAR_16Bitsto16Bits(F16toF16, vxc_half8,  vxc_half4,  half4)\n\
CROP_AND_RESIZE_BILINEAR_16Bitsto16Bits(F16toI16, vxc_half8,  vxc_short4, short4)\n\
\n\
#define CROP_AND_RESIZE_BILINEAR_F16toQuant8(name,dst_type) \\\n\
__kernel void crop_and_resize_bilinear_F16to##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    CROP_AND_RESIZE_PART0; \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        vxc_short8 src0, src1; \\\n\
        vxc_half8 src0_temp, src1_temp; \\\n\
        CROP_AND_RESIZE_PART1; \\\n\
 \\\n\
        _viv_asm(COPY, src0_temp, src0, 16); \\\n\
        _viv_asm(COPY, src1_temp, src1, 16); \\\n\
        float4 top, bottom, value; \\\n\
        float4 top_left4,top_right4,bottom_left4,bottom_right4; \\\n\
        dst_type data; \\\n\
        int4 tmpout; \\\n\
 \\\n\
        VXC_DP4x4(top_left4, src0_temp, src0_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(top_right4, src0_temp, src0_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_left4, src1_temp, src1_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_right4, src1_temp, src1_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
 \\\n\
        top = top_left4 + (top_right4 - top_left4) * x_lerp; \\\n\
        bottom = bottom_left4 + (bottom_right4 - bottom_left4) * x_lerp; \\\n\
        value = top + (bottom - top) * y_lerp; \\\n\
        value = value * inOutScale + inOutTile; \\\n\
        _viv_asm(CONV, tmpout, value); \\\n\
        VXC_DP2x8(data, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtract8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, data, \\\n\
                                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_BILINEAR_F16toQuant8(U8, vxc_uchar4)\n\
CROP_AND_RESIZE_BILINEAR_F16toQuant8(I8, vxc_char4)\n\
\n\
#define CROP_AND_RESIZE_BILINEAR_Quant8toF16(name,src_type) \\\n\
__kernel void crop_and_resize_bilinear_##name##toF16 \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    CROP_AND_RESIZE_PART0; \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        src_type src0, src1; \\\n\
        CROP_AND_RESIZE_PART1; \\\n\
 \\\n\
        float4 top, bottom, value; \\\n\
        float4 top_left4,top_right4,bottom_left4,bottom_right4; \\\n\
        vxc_half4 data; \\\n\
        vxc_short4 out; \\\n\
        half4 tmpout; \\\n\
 \\\n\
        VXC_DP4x4(top_left4, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(top_right4, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_left4, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniLeftToFp32_4x4); \\\n\
        VXC_DP4x4(bottom_right4, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightToFp32_4x4); \\\n\
 \\\n\
        top = top_left4 + (top_right4 - top_left4) * x_lerp; \\\n\
        bottom = bottom_left4 + (bottom_right4 - bottom_left4) * x_lerp; \\\n\
        value = top + (bottom - top) * y_lerp; \\\n\
        value = value * inOutScale + inOutTile; \\\n\
        _viv_asm(CONV, tmpout, value); \\\n\
        VXC_DP2x8(data, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8); \\\n\
        _viv_asm(COPY, out, data, 8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_BILINEAR_Quant8toF16(U8, vxc_uchar8)\n\
CROP_AND_RESIZE_BILINEAR_Quant8toF16(I8, vxc_char8)\n\
\n\
\n\
"; /* end of crop_and_resize_bilinear_vx*/

static const char crop_and_resize_nearest_neighbor_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float inOutScale;\n\
_viv_uniform float inOutTile;\n\
_viv_uniform float width_scale;\n\
_viv_uniform float height_scale;\n\
_viv_uniform int   image_width;\n\
_viv_uniform int   image_height;\n\
_viv_uniform VXC_512Bits uniConvertFstToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
\n\
#define IMG_LOAD(src_type) \\\n\
        src_type src; \\\n\
        int4 coord = (int4)(0, in_y, d + b * ori_depth, 0); \\\n\
        int8 input_desc, output_desc; \\\n\
 \\\n\
        coord_out.z = d + coord_out.z * ori_depth; \\\n\
 \\\n\
        _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
        int baseAddr = (int)coord.z * input_desc.s4 + input_desc.s0; \\\n\
        _viv_asm(MOV, coord.w, baseAddr); \\\n\
 \\\n\
        _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
        baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
        _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x + 1) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x + 2) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x + 3) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x + 4) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x + 5) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x + 6) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
        in_x = convert_int(round(xy.y * convert_float(image_width - 1) + convert_float(x + 7) * _width_scale)); \\\n\
        coord.x = in_x; \\\n\
        VXC_OP4(img_load_3d, src, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
\n\
#define CROP_AND_RESIZE_Quant8toQuant8(name, data_type) \\\n\
__kernel void crop_and_resize_nearest_neighbor_##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    int bb = get_global_id(2); \\\n\
    int y =  get_global_id(1); \\\n\
    int x = get_global_id(0); \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int2 coord_box_ind = (int2)(bb, 0); \\\n\
    int b = read_imagei(box_ind, coord_box_ind).x; \\\n\
    float4 xy; \\\n\
    int in_x, in_y; \\\n\
    int d = 0; \\\n\
    Image img_boxes = create_image_from_image2d(boxes, 2); \\\n\
    __global half* boxes_ptr = (__global half*)img_boxes.ptr; \\\n\
    xy = vload_half4(bb, boxes_ptr); \\\n\
    float _width_scale = convert_float(xy.w - xy.y) * width_scale; \\\n\
    float _height_scale = convert_float(xy.z - xy.x) * height_scale; \\\n\
 \\\n\
    if (_width_scale == 0) xy.y = 0.5 * (xy.y + xy.w); \\\n\
    if (_height_scale == 0) xy.x = 0.5 * (xy.x + xy.z); \\\n\
    in_y = convert_int(round(xy.x * convert_float(image_height - 1) + convert_float(y) * _height_scale)); \\\n\
 \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        data_type data; \\\n\
        int4 tmpout0, tmpout1; \\\n\
        float4 tmpdata0, tmpdata1; \\\n\
        IMG_LOAD(data_type); \\\n\
 \\\n\
        VXC_DP4x4(tmpdata0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
        VXC_DP4x4(tmpdata1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
 \\\n\
        tmpdata0 = tmpdata0 * inOutScale + inOutTile; \\\n\
        tmpdata1 = tmpdata1 * inOutScale + inOutTile; \\\n\
        _viv_asm(CONV, tmpout0, tmpdata0); \\\n\
        _viv_asm(CONV, tmpout1, tmpdata1); \\\n\
 \\\n\
        VXC_DP2x8(data, tmpout0, tmpout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Bit_2x8); \\\n\
 \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
                                     data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_Quant8toQuant8(U8toU8, vxc_uchar8)\n\
CROP_AND_RESIZE_Quant8toQuant8(I8toI8, vxc_char8)\n\
\n\
#define CROP_AND_RESIZE_Quant8toF16(name, src_type) \\\n\
__kernel void crop_and_resize_nearest_neighbor_##name##toF16 \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    int bb = get_global_id(2); \\\n\
    int y =  get_global_id(1); \\\n\
    int x = get_global_id(0); \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int2 coord_box_ind = (int2)(bb, 0); \\\n\
    int b = read_imagei(box_ind, coord_box_ind).x; \\\n\
    float4 xy; \\\n\
    int in_x, in_y; \\\n\
    int d = 0; \\\n\
    Image img_boxes = create_image_from_image2d(boxes, 2); \\\n\
    __global half* boxes_ptr = (__global half*)img_boxes.ptr; \\\n\
    xy = vload_half4(bb, boxes_ptr); \\\n\
    float _width_scale = convert_float(xy.w - xy.y) * width_scale; \\\n\
    float _height_scale = convert_float(xy.z - xy.x) * height_scale; \\\n\
 \\\n\
    if (_width_scale == 0) xy.y = 0.5 * (xy.y + xy.w); \\\n\
    if (_height_scale == 0) xy.x = 0.5 * (xy.x + xy.z); \\\n\
    in_y = convert_int(round(xy.x * convert_float(image_height - 1) + convert_float(y) * _height_scale)); \\\n\
 \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        vxc_short8 out; \\\n\
        vxc_half8 data; \\\n\
        half4 tmpout0, tmpout1; \\\n\
        float4 tmpdata0, tmpdata1; \\\n\
        IMG_LOAD(src_type); \\\n\
 \\\n\
        VXC_DP4x4(tmpdata0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
        VXC_DP4x4(tmpdata1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
 \\\n\
        tmpdata0 = tmpdata0 * inOutScale + inOutTile; \\\n\
        tmpdata1 = tmpdata1 * inOutScale + inOutTile; \\\n\
        _viv_asm(CONV, tmpout0, tmpdata0); \\\n\
        _viv_asm(CONV, tmpout1, tmpdata1); \\\n\
 \\\n\
        VXC_DP2x8(data, tmpout0, tmpout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8); \\\n\
        _viv_asm(COPY, out, data, 16); \\\n\
 \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, \\\n\
                                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_Quant8toF16(U8, vxc_uchar8)\n\
CROP_AND_RESIZE_Quant8toF16(I8, vxc_char8)\n\
\n\
#define CROP_AND_RESIZE_NEAREST_F16toQuant8(name, dst_type) \\\n\
__kernel void crop_and_resize_nearest_neighbor_F16to##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    int bb = get_global_id(2); \\\n\
    int y =  get_global_id(1); \\\n\
    int x = get_global_id(0); \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int2 coord_box_ind = (int2)(bb, 0); \\\n\
    int b = read_imagei(box_ind, coord_box_ind).x; \\\n\
    float4 xy; \\\n\
    int in_x, in_y; \\\n\
    int d = 0; \\\n\
    Image img_boxes = create_image_from_image2d(boxes, 2); \\\n\
    __global half* boxes_ptr = (__global half*)img_boxes.ptr; \\\n\
    xy = vload_half4(bb, boxes_ptr); \\\n\
    float _width_scale = convert_float(xy.w - xy.y) * width_scale; \\\n\
    float _height_scale = convert_float(xy.z - xy.x) * height_scale; \\\n\
 \\\n\
    if (_width_scale == 0) xy.y = 0.5 * (xy.y + xy.w); \\\n\
    if (_height_scale == 0) xy.x = 0.5 * (xy.x + xy.z); \\\n\
    in_y = convert_int(round(xy.x * convert_float(image_height - 1) + convert_float(y) * _height_scale)); \\\n\
 \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        dst_type data; \\\n\
        int4 tmpout0, tmpout1; \\\n\
        float4 tmpdata0, tmpdata1; \\\n\
        IMG_LOAD(vxc_short8); \\\n\
        vxc_half8 src_half; \\\n\
        _viv_asm(COPY, src_half, src, 16); \\\n\
 \\\n\
        VXC_DP4x4(tmpdata0, src_half, src_half, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
        VXC_DP4x4(tmpdata1, src_half, src_half, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
 \\\n\
        tmpdata0 = tmpdata0 * inOutScale + inOutTile; \\\n\
        tmpdata1 = tmpdata1 * inOutScale + inOutTile; \\\n\
        _viv_asm(CONV, tmpout0, tmpdata0); \\\n\
        _viv_asm(CONV, tmpout1, tmpdata1); \\\n\
 \\\n\
        VXC_DP2x8(data, tmpout0, tmpout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Bit_2x8); \\\n\
 \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_NEAREST_F16toQuant8(U8, vxc_uchar8)\n\
CROP_AND_RESIZE_NEAREST_F16toQuant8(I8, vxc_char8)\n\
\n\
#define CROP_AND_RESIZE_16Bitsto16Bits(name,src_type,dst_type,temp_type) \\\n\
__kernel void crop_and_resize_nearest_neighbor_##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout \\\n\
) \\\n\
{ \\\n\
    int bb = get_global_id(2); \\\n\
    int y =  get_global_id(1); \\\n\
    int x = get_global_id(0); \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int2 coord_box_ind = (int2)(bb, 0); \\\n\
    int b = read_imagei(box_ind, coord_box_ind).x; \\\n\
    float4 xy; \\\n\
    int in_x, in_y; \\\n\
    int d = 0; \\\n\
    Image img_boxes = create_image_from_image2d(boxes, 2); \\\n\
    __global half* boxes_ptr = (__global half*)img_boxes.ptr; \\\n\
    xy = vload_half4(bb, boxes_ptr); \\\n\
    float _width_scale = convert_float(xy.w - xy.y) * width_scale; \\\n\
    float _height_scale = convert_float(xy.z - xy.x) * height_scale; \\\n\
 \\\n\
    if (_width_scale == 0) xy.y = 0.5 * (xy.y + xy.w); \\\n\
    if (_height_scale == 0) xy.x = 0.5 * (xy.x + xy.z); \\\n\
    in_y = convert_int(round(xy.x * convert_float(image_height - 1) + convert_float(y) * _height_scale)); \\\n\
 \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        vxc_short8 out; \\\n\
        dst_type data; \\\n\
        temp_type tmpout0, tmpout1; \\\n\
        float4 tmpdata0, tmpdata1; \\\n\
        IMG_LOAD(vxc_short8); \\\n\
        src_type src_temp; \\\n\
        _viv_asm(COPY, src_temp, src, 16); \\\n\
 \\\n\
        VXC_DP4x4(tmpdata0, src_temp, src_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
        VXC_DP4x4(tmpdata1, src_temp, src_temp, \\\n\
                  VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
 \\\n\
        _viv_asm(CONV, tmpout0, tmpdata0); \\\n\
        _viv_asm(CONV, tmpout1, tmpdata1); \\\n\
 \\\n\
        VXC_DP2x8(data, tmpout0, tmpout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8); \\\n\
        _viv_asm(COPY, out, data, 16); \\\n\
 \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CROP_AND_RESIZE_16Bitsto16Bits \\\n\
(F16toF16, vxc_half8, vxc_half8,  half4)\n\
CROP_AND_RESIZE_16Bitsto16Bits \\\n\
(F16toI16, vxc_half8, vxc_short8, short4)\n\
CROP_AND_RESIZE_16Bitsto16Bits \\\n\
(I16toF16, vxc_short8, vxc_half8, half4)\n\
CROP_AND_RESIZE_16Bitsto16Bits \\\n\
(I16toI16, vxc_short8, vxc_short8,short4)\n\
"; /* end of crop_and_resize_nearest_neighbor_vx*/

static const char cumsum_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32B_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
\n\
__kernel void cumsum_F16toF16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_AXIS2(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_##in_name##to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
 \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
        VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
        VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
        float tmpAlpha = convert_float(coord.z + 1) * in_out_zp_scale + output_zp; \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
        int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_AXIS2(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_AXIS2(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void cumsum_I16toI16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
        float tmpAlpha = convert_float(coord.z + 1) * in_out_zp_scale + output_zp;\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1), uniConvertInt32toUint8_2x8);\n\
\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void cumsum_F16toF16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_AXIS1(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_##in_name##to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
        VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
        VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp; \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
        int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_AXIS1(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_AXIS1(I8, I8, vxc_char16,  vxc_char16)\n\
\n\
__kernel void cumsum_I16toI16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp;\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void cumsum_F16toF16_axis0(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, tmpsum, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16A_4x4);\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16B_4x4);\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16C_2x8);\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define CUMSUM_QINT_AXIS0(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_##in_name##to##out_name##_axis0( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    vxc_short8 rowSum; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0); \\\n\
    short zp = (short)input_zp; \\\n\
 \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16A_4x4); \\\n\
        VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16B_8x4); \\\n\
        VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpI16toI16_2x8); \\\n\
        VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumHorzI16toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumHorzI16toI32B_4x4); \\\n\
 \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
\n\
CUMSUM_QINT_AXIS0(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
CUMSUM_QINT_AXIS0(I8,  I8,  vxc_char16,  vxc_char16)\n\
CUMSUM_QINT_AXIS0(I16, I16, vxc_short8,  vxc_short8)\n\
"; /* end of cumsum_vx*/

static const char cumsum_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32B_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
\n\
__kernel void cumsum_F16toF16_axis1_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
\n\
    for(; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_AXIS1_2D(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_##in_name##to##out_name##_axis1_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0); \\\n\
    int4 sum1 = (int4)(0); \\\n\
    int4 sum2 = (int4)(0); \\\n\
    int4 sum3 = (int4)(0); \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32B_4x4); \\\n\
        VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32C_4x4); \\\n\
        VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32D_4x4); \\\n\
 \\\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp; \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
        int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                 uniConvertInt32toUint8_2x8); \\\n\
        VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), \\\n\
                 uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
\n\
CUMSUM_8BITS_AXIS1_2D(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_AXIS1_2D(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void cumsum_I16toI16_axis1_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32A_4x4);\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32B_4x4);\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp;\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                 uniConvertInt32toUint8_2x8);\n\
\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void cumsum_F16toF16_axis0_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, tmpsum, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16A_4x4);\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16B_4x4);\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16C_2x8);\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define CUMSUM_QINT_AXIS0_2D(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_##in_name##to##out_name##_axis0_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    vxc_short8 rowSum; \\\n\
    int4 sum0, sum1; \\\n\
    sum0 ^= sum0; \\\n\
    sum1 ^= sum1; \\\n\
    short zp = (short)input_zp; \\\n\
 \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzU8toI16A_4x4); \\\n\
        VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzU8toI16B_8x4); \\\n\
        VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSubZpI16toI16_2x8); \\\n\
        VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzI16toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzI16toI32B_4x4); \\\n\
 \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
 \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                 uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
\n\
CUMSUM_QINT_AXIS0_2D(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
CUMSUM_QINT_AXIS0_2D(I8,  I8,  vxc_char16,  vxc_char16)\n\
CUMSUM_QINT_AXIS0_2D(I16, I16, vxc_short8,  vxc_short8)\n\
"; /* end of cumsum_2d_vx*/

static const char cumsum_array_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32B_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
__kernel void cumsum_array_F16toF16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_ARRAY_AXIS2(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_array_##in_name##to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
 \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 1); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 1); \\\n\
    if (coord.x == ((w_size >> 4) * 16) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (16 - remainder); \\\n\
    } \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
        VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
        VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
        float tmpAlpha = convert_float(coord.z + 1) * in_out_zp_scale + output_zp; \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
        int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1), uniConvertInt32toUint8_2x8);\\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_ARRAY_AXIS2(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_ARRAY_AXIS2(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void cumsum_array_I16toI16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
        float tmpAlpha = convert_float(coord.z + 1) * in_out_zp_scale + output_zp;\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1), uniConvertInt32toUint8_2x8);\n\
\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
__kernel void cumsum_array_F16toF16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_ARRAY_AXIS1(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_array_##in_name##to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2); \\\n\
    if (coord.x == ((w_size >> 4) * 16) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (16 - remainder); \\\n\
    } \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
        VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
        VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp; \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
        int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_ARRAY_AXIS1(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_ARRAY_AXIS1(I8, I8, vxc_char16,  vxc_char16)\n\
\n\
__kernel void cumsum_array_I16toI16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp;\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
__kernel void cumsum_array_F16toF16_axis0(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, tmpsum, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
        {\n\
            coord.x = coord.x - (8 - remainder);\n\
        }\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16A_4x4);\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16B_4x4);\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16C_2x8);\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        out_ptr[0] = dst;\n\
\n\
    }\n\
}\n\
\n\
#define CUMSUM_ARRAY_QINT_AXIS0(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_array_##in_name##to##out_name##_axis0( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    vxc_short8 rowSum; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0); \\\n\
    short zp = (short)input_zp; \\\n\
 \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
        { \\\n\
            coord.x = coord.x - (8 - remainder); \\\n\
        } \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16A_4x4); \\\n\
        VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16B_8x4); \\\n\
        VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpI16toI16_2x8); \\\n\
        VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumHorzI16toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumHorzI16toI32B_4x4); \\\n\
 \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
\n\
CUMSUM_ARRAY_QINT_AXIS0(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
CUMSUM_ARRAY_QINT_AXIS0(I8,  I8,  vxc_char16,  vxc_char16)\n\
CUMSUM_ARRAY_QINT_AXIS0(I16, I16, vxc_short8,  vxc_short8)\n\
"; /* end of cumsum_array_vx*/

static const char cumsum_array_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32B_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
__kernel void cumsum_array_F16toF16_axis1_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
\n\
    Image img1 = create_image_from_image2d(input, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    for(; coord.y < height; coord.y++)\n\
    {\n\
        uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_ARRAY_AXIS1_2D(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_array_##in_name##to##out_name##_axis1_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0); \\\n\
    int4 sum1 = (int4)(0); \\\n\
    int4 sum2 = (int4)(0); \\\n\
    int4 sum3 = (int4)(0); \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input, 1); \\\n\
    Image img2 = create_image_from_image2d(output, 1); \\\n\
    if (coord.x == ((w_size >> 4) * 16) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (16 - remainder); \\\n\
    } \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        uchar* input_ptr = get_image_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_image_ptr_from_coord(img2, coord); \\\n\
        __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32B_4x4); \\\n\
        VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32C_4x4); \\\n\
        VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32D_4x4); \\\n\
 \\\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp; \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
        float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
        int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
        int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
 \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                 uniConvertInt32toUint8_2x8); \\\n\
        VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), \\\n\
                 uniConvertInt32toUint8_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_ARRAY_AXIS1_2D(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_ARRAY_AXIS1_2D(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void cumsum_array_I16toI16_axis1_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(input, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32A_4x4);\n\
        VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertU8toI32B_4x4);\n\
        float tmpAlpha = convert_float(coord.y + 1) * in_out_zp_scale + output_zp;\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                 uniConvertInt32toUint8_2x8);\n\
\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
__kernel void cumsum_array_F16toF16_axis0_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, tmpsum, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    Image img1 = create_image_from_image2d(input, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
        {\n\
            coord.x = coord.x - (8 - remainder);\n\
        }\n\
        uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
        __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
        src = in_ptr[0];\n\
        _viv_asm(COPY, data, src, 16);\n\
\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16A_4x4);\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16B_4x4);\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16C_2x8);\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzF16toF16_2x8);\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
#define CUMSUM_ARRAY_QINT_AXIS0_2D(in_name, out_name, src_type, dst_type, stride_data) \\\n\
__kernel void cumsum_array_##in_name##to##out_name##_axis0_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    vxc_short8 rowSum; \\\n\
    int4 sum0, sum1; \\\n\
    sum0 ^= sum0; \\\n\
    sum1 ^= sum1; \\\n\
    short zp = (short)input_zp; \\\n\
    Image img1 = create_image_from_image2d(input, stride_data); \\\n\
    Image img2 = create_image_from_image2d(output, stride_data); \\\n\
 \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
        { \\\n\
            coord.x = coord.x - (8 - remainder); \\\n\
        } \\\n\
        uchar* input_ptr = get_image_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_image_ptr_from_coord(img2, coord); \\\n\
        __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzU8toI16A_4x4); \\\n\
        VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzU8toI16B_8x4); \\\n\
        VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSubZpI16toI16_2x8); \\\n\
        VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzI16toI32A_4x4); \\\n\
        VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzI16toI32B_4x4); \\\n\
 \\\n\
        float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
        float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
        int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
        int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
 \\\n\
        VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                 uniConvertInt32toUint8_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
\n\
CUMSUM_ARRAY_QINT_AXIS0_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, 1)\n\
CUMSUM_ARRAY_QINT_AXIS0_2D(I8,  I8,  vxc_char16,  vxc_char16, 1)\n\
CUMSUM_ARRAY_QINT_AXIS0_2D(I16, I16, vxc_short8,  vxc_short8, 2)"; /* end of cumsum_array_2d_vx*/

static const char cumsum_array_bf16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
__kernel void cumsum_array_BF16toBF16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 sum0 = (float4)(0), sum1 = (float4)(0);\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        float4 data0, data1;\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_ushort8* in_ptr = (__global vxc_ushort8*)input_ptr;\n\
        __global vxc_ushort8* out_ptr = (__global vxc_ushort8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        sum0 += data0;\n\
        sum1 += data1;\n\
        _viv_asm(COPY, dst0, sum0, 16);\n\
        _viv_asm(COPY, dst1, sum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 sum0 = (float4)(0), sum1 = (float4)(0);\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        float4 data0, data1;\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_ushort8* in_ptr = (__global vxc_ushort8*)input_ptr;\n\
        __global vxc_ushort8* out_ptr = (__global vxc_ushort8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
        sum0 += data0;\n\
        sum1 += data1;\n\
        _viv_asm(COPY, dst0, sum0, 16);\n\
        _viv_asm(COPY, dst1, sum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis0(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float preSum = 0;\n\
    float4 one = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    float4 q = (float4)(1.0, 1.0, 1.0, 0);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        float4 data0, data1;\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
        {\n\
            coord.x = coord.x - (8 - remainder);\n\
        }\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        __global vxc_ushort8* in_ptr = (__global vxc_ushort8*)input_ptr;\n\
        __global vxc_ushort8* out_ptr = (__global vxc_ushort8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        float4 tmpSum0 = (float4)(data0.x, data0.x + data0.y, dot(data0, q), dot(data0, one));\n\
        float4 tmpSum1 = (float4)(data1.x, data1.x + data1.y, dot(data1, q), dot(data1, one));\n\
        tmpSum1 += tmpSum0.w;\n\
\n\
        tmpSum0 += preSum;\n\
        tmpSum1 += preSum;\n\
\n\
        preSum = tmpSum1.w;\n\
\n\
        _viv_asm(COPY, dst0, tmpSum0, 16);\n\
        _viv_asm(COPY, dst1, tmpSum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis1_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 sum0 = (float4)(0), sum1 = (float4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(input, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
\n\
    for(; coord.y < height; coord.y++)\n\
    {\n\
        float4 data0, data1;\n\
        uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
        __global vxc_ushort8* in_ptr = (__global vxc_ushort8*)input_ptr;\n\
        __global vxc_ushort8* out_ptr = (__global vxc_ushort8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        sum0 += data0;\n\
        sum1 += data1;\n\
\n\
        _viv_asm(COPY, dst0, sum0, 16);\n\
        _viv_asm(COPY, dst1, sum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniExtractOddData_2x8);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis0_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float preSum = 0;\n\
    float4 one = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    float4 q = (float4)(1.0, 1.0, 1.0, 0);\n\
\n\
    Image img1 = create_image_from_image2d(input, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
        {\n\
            coord.x = coord.x - (8 - remainder);\n\
        }\n\
        float4 data0, data1;\n\
        uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
        uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
        __global vxc_ushort8* in_ptr = (__global vxc_ushort8*)input_ptr;\n\
        __global vxc_ushort8* out_ptr = (__global vxc_ushort8*)output_ptr;\n\
        src = in_ptr[0];\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        float4 tmpSum0 = (float4)(data0.x, data0.x + data0.y, dot(data0, q), dot(data0, one));\n\
        float4 tmpSum1 = (float4)(data1.x, data1.x + data1.y, dot(data1, q), dot(data1, one));\n\
        tmpSum1 += tmpSum0.w;\n\
\n\
        tmpSum0 += preSum;\n\
        tmpSum1 += preSum;\n\
\n\
        preSum = tmpSum1.w;\n\
\n\
        _viv_asm(COPY, dst0, tmpSum0, 16);\n\
        _viv_asm(COPY, dst1, tmpSum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniExtractOddData_2x8);\n\
        out_ptr[0] = dst;\n\
    }\n\
}\n\
"; /* end of cumsum_array_bf16_vx*/

static const char cumsum_array_ex_rev_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32B_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzRevF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzRevF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzRevF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzRevF16toF16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzRevU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzRevU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpRevI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzRevI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzRevI16toI32B_4x4;\n\
\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
__kernel void cumsum_ex_rev_array_F16toF16_axis0(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, tmpsum, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
    __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.x = width - 8; coord.x >= 0; coord.x -= 8)\n\
        {\n\
            if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
            {\n\
                coord.x = coord.x - (8 - remainder);\n\
            }\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            src = in_ptr[0];\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16A_4x4);\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16B_4x4);\n\
            VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniSumHorzRevF16toF16C_2x8);\n\
            VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzRevF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        out_ptr[0] = dst;\n\
        for(; coord.x < width - 8;)\n\
        {\n\
            if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
            {\n\
                coord.x = coord.x - (8 - remainder);\n\
            }\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            coord_out.x = coord.x + 1;\n\
            coord.x += 8;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16A_4x4);\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16B_4x4);\n\
            VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16C_2x8);\n\
            VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        coord.x = width - 8;\n\
        coord_out.x = width - 1;\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
        out_ptr = (__global vxc_short8*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(; coord.x > 0;)\n\
        {\n\
            if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
            {\n\
                coord.x = coord.x - (8 - remainder);\n\
            }\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            src = in_ptr[0];\n\
            coord_out.x = coord.x - 1;\n\
            coord.x -= 8;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16A_4x4);\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16B_4x4);\n\
            VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniSumHorzRevF16toF16C_2x8);\n\
            VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzRevF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_QINT_EX_REV_ARRAY_AXIS0(in_name, out_name, src_type, dst_type, stride_data) \\\n\
__kernel void cumsum_ex_rev_array_##in_name##to##out_name##_axis0( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    vxc_short8 rowSum; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0); \\\n\
    short zp = (short)input_zp; \\\n\
 \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, stride_data); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_data); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
    __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.x = width - 8; coord.x >= 0; coord.x -= 8) \\\n\
        { \\\n\
            if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
            { \\\n\
                coord.x = coord.x - (8 - remainder); \\\n\
            } \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16A_4x4); \\\n\
            VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16B_8x4); \\\n\
            VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpRevI16toI16_2x8); \\\n\
            VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32B_4x4); \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            VXC_DP2x8(dst, tmpDst1, tmpDst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        for(coord.x = -1; coord.x < width - 8;) \\\n\
        { \\\n\
            if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
            { \\\n\
                coord.x = coord.x - (8 - remainder); \\\n\
            } \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord_out.x = coord.x + 1; \\\n\
            coord.x += 8; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16A_4x4); \\\n\
            VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16B_8x4); \\\n\
            VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpI16toI16_2x8); \\\n\
            VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzI16toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzI16toI32B_4x4); \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        for(coord.x = width - 7; coord.x > 0;) \\\n\
        { \\\n\
            if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
            { \\\n\
                coord.x = coord.x - (8 - remainder); \\\n\
            } \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord_out.x = coord.x - 1; \\\n\
            coord.x -= 8; \\\n\
            VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16A_4x4); \\\n\
            VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16B_8x4); \\\n\
            VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpRevI16toI16_2x8); \\\n\
            VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32B_4x4); \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            VXC_DP2x8(dst, tmpDst1, tmpDst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_QINT_EX_REV_ARRAY_AXIS0(U8,  U8,  vxc_uchar16, vxc_uchar16, 1)\n\
CUMSUM_QINT_EX_REV_ARRAY_AXIS0(I8,  I8,  vxc_char16,  vxc_char16, 1)\n\
CUMSUM_QINT_EX_REV_ARRAY_AXIS0(I16, I16, vxc_short8,  vxc_short8, 2)\n\
"; /* end of cumsum_array_ex_rev_axis0_vx*/

static const char cumsum_array_ex_rev_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int height;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
__kernel void cumsum_ex_rev_array_F16toF16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
    __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            src = in_ptr[0];\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        dst ^= dst;\n\
        out_ptr[0] = dst;\n\
        for(; coord.y < height - 1;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            coord.y++;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        dst ^= dst;\n\
        coord.y = height - 1;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        out_ptr = (__global vxc_short8*)output_ptr;\n\
        out_ptr[0] = dst;\n\
\n\
        for(; coord.y > 0;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            coord.y--;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_EX_REV_ARRAY_AXIS1(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_ex_rev_array_##in_name##to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
 \\\n\
    if (coord.x == ((w_size >> 4) * 16) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (16 - remainder); \\\n\
    } \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 1); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 1); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
    __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        out_ptr[0] = dst.xxxxxxxxxxxxxxxx; \\\n\
        for(; coord.y < height - 1;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord.y++; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(coord.y) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        coord.y = height - 1; \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        out_ptr = (__global vxc_short8*)output_ptr; \\\n\
        out_ptr[0] = dst.xxxxxxxxxxxxxxxx; \\\n\
        for(; coord.y > 0;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp; \\\n\
            coord.y--; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_EX_REV_ARRAY_AXIS1(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_EX_REV_ARRAY_AXIS1(I8, I8, vxc_char16,  vxc_char16)\n\
\n\
__kernel void cumsum_ex_rev_array_I16toI16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
    __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            src = in_ptr[0];\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                        uniConvertInt32toUint8_2x8);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        out_ptr[0] = dst.xxxxxxxx;\n\
        for(; coord.y < height - 1;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            coord.y++;\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(coord.y) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        coord.y = height - 1;\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        out_ptr = (__global vxc_short8*)output_ptr;\n\
        out_ptr[0] = dst.xxxxxxxx;\n\
        for(; coord.y > 0;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp;\n\
            coord.y--;\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
"; /* end of cumsum_array_ex_rev_axis1_vx*/

static const char cumsum_array_ex_rev_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int channel;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
__kernel void cumsum_ex_rev_array_F16toF16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
    __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
    if(rev && exclusive == 0)\n\
    {\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            src = in_ptr[0];\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(rev == 0 && exclusive)\n\
    {\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        out_ptr[0] = dst;\n\
        for(; coord.z < channel - 1;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            coord.z++;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(rev && exclusive)\n\
    {\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        coord.z = channel - 1;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        out_ptr = (__global vxc_short8*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(; coord.z > 0;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            coord.z--;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_EX_REV_ARRAY_AXIS2(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_ex_rev_array_##in_name##to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
 \\\n\
    if (coord.x == ((w_size >> 4) * 16) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (16 - remainder); \\\n\
    } \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 1); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 1); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
    __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
    if(rev && exclusive == 0) \\\n\
    { \\\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
        out_ptr[0] = dst.xxxxxxxxxxxxxxxx; \\\n\
        for(; coord.z < channel - 1;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord.z++; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(coord.z) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(rev && exclusive) \\\n\
    { \\\n\
        coord.z = channel - 1; \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        out_ptr = (__global vxc_short8*)output_ptr; \\\n\
        out_ptr[0] = dst.xxxxxxxxxxxxxxxx; \\\n\
        for(; coord.z > 0;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp; \\\n\
            coord.z--; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_EX_REV_ARRAY_AXIS2(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_EX_REV_ARRAY_AXIS2(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void cumsum_ex_rev_array_I16toI16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0)\n\
    {\n\
        coord.x = coord.x - (8 - remainder);\n\
    }\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr;\n\
    __global vxc_short8* out_ptr = (__global vxc_short8*)output_ptr;\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            out_ptr = (__global vxc_short8*)output_ptr;\n\
            src = in_ptr[0];\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        out_ptr[0] = dst.xxxxxxxx;\n\
        for(; coord.z < channel - 1;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            coord.z++;\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(coord.z) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        coord.z = channel - 1;\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
        out_ptr = (__global vxc_short8*)output_ptr;\n\
        out_ptr[0] = dst.xxxxxxxx;\n\
        for(; coord.z > 0;)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global vxc_short8*)input_ptr;\n\
            src = in_ptr[0];\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp;\n\
            coord.z--;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
\n\
"; /* end of cumsum_array_ex_rev_axis2_vx*/

static const char cumsum_array_f16_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
#define CUMSUM_ARRAY_F16TOQINT_AXIS2(out_name, src_type, dst_type, stride_out) \\\n\
__kernel void cumsum_array_F16to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_out); \\\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (8 - remainder); \\\n\
    } \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_F16TOQINT_AXIS2(I8,  vxc_half8, vxc_char16, 1)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS2(I16, vxc_half8, vxc_short8, 2)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS2(U8,  vxc_half8, vxc_uchar16, 1)\n\
\n\
\n\
#define CUMSUM_ARRAY_F16TOQINT_AXIS1(out_name, src_type, dst_type, stride_out) \\\n\
__kernel void cumsum_array_F16to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_out); \\\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (8 - remainder); \\\n\
    } \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_F16TOQINT_AXIS1(I8,  vxc_half8, vxc_char16, 1)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS1(I16, vxc_half8, vxc_short8, 2)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS1(U8,  vxc_half8, vxc_uchar16, 1)\n\
\n\
#define CUMSUM_ARRAY_F16TOQINT_AXIS0(out_name, src_type, dst_type, stride_out) \\\n\
__kernel void cumsum_array_F16to##out_name##_axis0( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, tmpsum, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_out); \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
        { \\\n\
            coord.x = coord.x - (8 - remainder); \\\n\
        } \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16A_4x4); \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16B_4x4); \\\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16C_2x8); \\\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_F16TOQINT_AXIS0(I8,  vxc_half8, vxc_char16, 1)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS0(I16, vxc_half8, vxc_short8, 2)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS0(U8,  vxc_half8, vxc_uchar16, 1)\n\
\n\
#define CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS2(out_name, src_type, dst_type, stride_out) \\\n\
__kernel void cumsum_array_ex_rev_F16to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (8 - remainder); \\\n\
    } \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_out); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr; \\\n\
    __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            in_ptr = (__global vxc_short8*)input_ptr; \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
        for(; coord.z < channel - 1;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global vxc_short8*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord.z++; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
     \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        coord.z = channel - 1; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        out_ptr = (__global dst_type*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
        for(; coord.z > 0;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global vxc_short8*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord.z--; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
     \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS2(I8,  vxc_half8, vxc_char16, 1)\n\
CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS2(I16, vxc_half8, vxc_short8, 2)\n\
CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS2(U8,  vxc_half8, vxc_uchar16, 1)\n\
\n\
#define CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS1(out_name, src_type, dst_type, stride_out) \\\n\
__kernel void cumsum_array_ex_rev_F16to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (8 - remainder); \\\n\
    } \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_out); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr; \\\n\
    __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            in_ptr = (__global vxc_short8*)input_ptr; \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
        for(; coord.y < height - 1;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global vxc_short8*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord.y++; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        coord.y = height - 1; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        out_ptr = (__global dst_type*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
        for(; coord.y > 0;) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global vxc_short8*)input_ptr; \\\n\
            src = in_ptr[0]; \\\n\
            coord.y--; \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global dst_type*)output_ptr; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS1(I8,  vxc_half8, vxc_char16, 1)\n\
CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS1(I16, vxc_half8, vxc_short8, 2)\n\
CUMSUM_ARRAY_F16TOQINT_EX_REV_AXIS1(U8,  vxc_half8, vxc_uchar16, 1)"; /* end of cumsum_array_f16_u8_vx*/

static const char cumsum_array_f16_u8_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform int remainder;\n\
_viv_uniform int w_size;\n\
\n\
\n\
#define CUMSUM_ARRAY_F16TOQINT_AXIS1_2D(out_name, src_type, dst_type, stride_out) \\\n\
__kernel void cumsum_array_F16to##out_name##_axis1_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_out); \\\n\
    if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
    { \\\n\
        coord.x = coord.x - (8 - remainder); \\\n\
    } \\\n\
    for(; coord.y < height; coord.y++) \\\n\
    { \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_F16TOQINT_AXIS1_2D(I8,  vxc_half8, vxc_char16, 1)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS1_2D(I16, vxc_half8, vxc_short8, 2)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS1_2D(U8,  vxc_half8, vxc_uchar16, 1)\n\
\n\
#define CUMSUM_ARRAY_F16TOQINT_AXIS0_2D(out_name, src_type, dst_type, stride_out) \\\n\
__kernel void cumsum_array_F16to##out_name##_axis0_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, tmpsum, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 2); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, stride_out); \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        if (coord.x == ((w_size >> 3) * 8) && remainder != 0) \\\n\
        { \\\n\
            coord.x = coord.x - (8 - remainder); \\\n\
        } \\\n\
        uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
        uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
        __global vxc_short8* in_ptr = (__global vxc_short8*)input_ptr; \\\n\
        __global dst_type* out_ptr = (__global dst_type*)output_ptr; \\\n\
        src = in_ptr[0]; \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16A_4x4); \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16B_4x4); \\\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16C_2x8); \\\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        out_ptr[0] = dst; \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_F16TOQINT_AXIS0_2D(I8,  vxc_half8, vxc_char16, 1)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS0_2D(I16, vxc_half8, vxc_short8, 2)\n\
CUMSUM_ARRAY_F16TOQINT_AXIS0_2D(U8,  vxc_half8, vxc_uchar16, 1)\n\
"; /* end of cumsum_array_f16_u8_2d_vx*/

static const char cumsum_bf16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
\n\
__kernel void cumsum_BF16toBF16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 sum0 = (float4)(0), sum1 = (float4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        float4 data0, data1;\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        sum0 += data0;\n\
        sum1 += data1;\n\
        _viv_asm(COPY, dst0, sum0, 16);\n\
        _viv_asm(COPY, dst1, sum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 sum0 = (float4)(0), sum1 = (float4)(0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        float4 data0, data1;\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
        sum0 += data0;\n\
        sum1 += data1;\n\
        _viv_asm(COPY, dst0, sum0, 16);\n\
        _viv_asm(COPY, dst1, sum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis0(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float preSum = 0;\n\
    float4 one = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    float4 q = (float4)(1.0, 1.0, 1.0, 0);\n\
\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        float4 data0, data1;\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        float4 tmpSum0 = (float4)(data0.x, data0.x + data0.y, dot(data0, q), dot(data0, one));\n\
        float4 tmpSum1 = (float4)(data1.x, data1.x + data1.y, dot(data1, q), dot(data1, one));\n\
        tmpSum1 += tmpSum0.w;\n\
\n\
        tmpSum0 += preSum;\n\
        tmpSum1 += preSum;\n\
\n\
        preSum = tmpSum1.w;\n\
\n\
        _viv_asm(COPY, dst0, tmpSum0, 16);\n\
        _viv_asm(COPY, dst1, tmpSum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis1_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 sum0 = (float4)(0), sum1 = (float4)(0);\n\
\n\
    for(; coord.y < height; coord.y++)\n\
    {\n\
        float4 data0, data1;\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        sum0 += data0;\n\
        sum1 += data1;\n\
\n\
        _viv_asm(COPY, dst0, sum0, 16);\n\
        _viv_asm(COPY, dst1, sum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void cumsum_BF16toBF16_axis0_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_ushort8 src, val0, val1;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float preSum = 0;\n\
    float4 one = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    float4 q = (float4)(1.0, 1.0, 1.0, 0);\n\
\n\
    for(; coord.x < width; coord.x += 8)\n\
    {\n\
        float4 data0, data1;\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(val0, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(val1, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, val0, 16);\n\
        _viv_asm(COPY, data1, val1, 16);\n\
\n\
        float4 tmpSum0 = (float4)(data0.x, data0.x + data0.y, dot(data0, q), dot(data0, one));\n\
        float4 tmpSum1 = (float4)(data1.x, data1.x + data1.y, dot(data1, q), dot(data1, one));\n\
        tmpSum1 += tmpSum0.w;\n\
\n\
        tmpSum0 += preSum;\n\
        tmpSum1 += preSum;\n\
\n\
        preSum = tmpSum1.w;\n\
\n\
        _viv_asm(COPY, dst0, tmpSum0, 16);\n\
        _viv_asm(COPY, dst1, tmpSum1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of cumsum_bf16_vx*/

static const char cumsum_ex_rev_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzI16toI32B_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzRevF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzRevF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzRevF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzRevF16toF16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumHorzRevU8toI16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzRevU8toI16B_8x4;\n\
_viv_uniform VXC_512Bits uniSubZpRevI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzRevI16toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumHorzRevI16toI32B_4x4;\n\
\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float output_zp;\n\
\n\
__kernel void cumsum_ex_rev_F16toF16_axis0(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, tmpsum, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.x = width - 8; coord.x >= 0; coord.x -= 8)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16A_4x4);\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16B_4x4);\n\
            VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniSumHorzRevF16toF16C_2x8);\n\
            VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzRevF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        for(; coord.x < width - 8;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord_out.x = coord.x + 1;\n\
            coord.x += 8;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16A_4x4);\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16B_4x4);\n\
            VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16C_2x8);\n\
            VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        coord.x = width - 8;\n\
        coord_out.x = width - 1;\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        for(; coord.x > 0;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord_out.x = coord.x - 1;\n\
            coord.x -= 8;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16A_4x4);\n\
            VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevF16toF16B_4x4);\n\
            VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniSumHorzRevF16toF16C_2x8);\n\
            VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzRevF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_QINT_EX_REV_AXIS0(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_ex_rev_##in_name##to##out_name##_axis0( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    vxc_short8 rowSum; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0); \\\n\
    short zp = (short)input_zp; \\\n\
 \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.x = width - 8; coord.x >= 0; coord.x -= 8) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16A_4x4); \\\n\
            VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16B_8x4); \\\n\
            VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpRevI16toI16_2x8); \\\n\
            VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32B_4x4); \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            VXC_DP2x8(dst, tmpDst1, tmpDst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        for(coord.x = -1; coord.x < width - 8;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord_out.x = coord.x + 1; \\\n\
            coord.x += 8; \\\n\
            VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16A_4x4); \\\n\
            VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzU8toI16B_8x4); \\\n\
            VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpI16toI16_2x8); \\\n\
            VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzI16toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzI16toI32B_4x4); \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        for(coord.x = width - 7; coord.x > 0;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord_out.x = coord.x - 1; \\\n\
            coord.x -= 8; \\\n\
            VXC_DP4x4(rowSum, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16A_4x4); \\\n\
            VXC_DP8x4(rowSum, src, src, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzRevU8toI16B_8x4); \\\n\
            VXC_DP2x8(rowSum, rowSum, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSubZpRevI16toI16_2x8); \\\n\
            VXC_DP4x4(sum0, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, rowSum, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniAccSumHorzRevI16toI32B_4x4); \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + output_zp; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + output_zp; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            VXC_DP2x8(dst, tmpDst1, tmpDst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_QINT_EX_REV_AXIS0(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
CUMSUM_QINT_EX_REV_AXIS0(I8,  I8,  vxc_char16,  vxc_char16)\n\
CUMSUM_QINT_EX_REV_AXIS0(I16, I16, vxc_short8,  vxc_short8)\n\
"; /* end of cumsum_ex_rev_axis0_vx*/

static const char cumsum_ex_rev_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int height;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
\n\
__kernel void cumsum_ex_rev_F16toF16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        dst ^= dst;\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        for(; coord.y < height - 1;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        dst ^= dst;\n\
        coord.y = height - 1;\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        for(; coord.y > 0;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y--;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_EX_REV_AXIS1(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_ex_rev_##in_name##to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
 \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxxxxxxxxxx, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.y < height - 1;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(coord.y) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        coord.y = height - 1; \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxxxxxxxxxx, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.y > 0;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp; \\\n\
            coord.y--; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15,0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_EX_REV_AXIS1(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_EX_REV_AXIS1(I8, I8, vxc_char16,  vxc_char16)\n\
\n\
__kernel void cumsum_ex_rev_I16toI16_axis1(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxx, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        for(; coord.y < height - 1;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(coord.y) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        coord.y = height - 1;\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxx, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        for(; coord.y > 0;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(height - coord.y) * in_out_zp_scale + output_zp;\n\
            coord.y--;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
}\n\
"; /* end of cumsum_ex_rev_axis1_vx*/

static const char cumsum_ex_rev_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32A_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32B_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32C_4x4;\n\
_viv_uniform VXC_512Bits uniAccSumVertU8toI32D_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int channel;\n\
_viv_uniform float in_out_scale;\n\
_viv_uniform float in_out_zp_scale;\n\
_viv_uniform float output_zp;\n\
\n\
__kernel void cumsum_ex_rev_F16toF16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_short8 src, dst;\n\
    vxc_half8 data, sum;\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8);\n\
    if(rev && exclusive == 0)\n\
    {\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(rev == 0 && exclusive)\n\
    {\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        for(; coord.z < channel - 1;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.z++;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(rev && exclusive)\n\
    {\n\
        _viv_asm(COPY, dst, sum, 16);\n\
        coord.z = channel - 1;\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        for(; coord.z > 0;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.z--;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8);\n\
            _viv_asm(COPY, dst, sum, 16);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_8BITS_EX_REV_AXIS2(in_name, out_name, src_type, dst_type) \\\n\
__kernel void cumsum_ex_rev_##in_name##to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0), sum2 = (int4)(0), sum3 = (int4)(0); \\\n\
 \\\n\
    if(rev && exclusive == 0) \\\n\
    { \\\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1), \\\n\
                        uniConvertInt32toUint8_2x8);\\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxxxxxxxxxx, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.z < channel - 1;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.z++; \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(coord.z) * in_out_zp_scale + output_zp; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(rev && exclusive) \\\n\
    { \\\n\
        coord.z = channel - 1; \\\n\
        int tmpAlpha0 = convert_int_rte(output_zp); \\\n\
        int4 tmpVal; \\\n\
        tmpVal.x = tmpAlpha0; \\\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\\\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxxxxxxxxxx, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.z > 0;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4); \\\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4); \\\n\
            VXC_DP4x4(sum2, src, sum2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32C_4x4); \\\n\
            VXC_DP4x4(sum3, src, sum3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32D_4x4); \\\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp; \\\n\
            coord.z--; \\\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum2 = convert_float4(sum2) * in_out_scale + tmpAlpha; \\\n\
            float4 tmpSum3 = convert_float4(sum3) * in_out_scale + tmpAlpha; \\\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0); \\\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1); \\\n\
            int4 tmpDst2 = convert_int4_rte(tmpSum2); \\\n\
            int4 tmpDst3 = convert_int4_rte(tmpSum3); \\\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_DP2x8(dst, tmpDst2, tmpDst3, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_8BITS_EX_REV_AXIS2(U8, U8, vxc_uchar16, vxc_uchar16)\n\
CUMSUM_8BITS_EX_REV_AXIS2(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void cumsum_ex_rev_I16toI16_axis2(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    int axis, int exclusive, int rev)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_short8 src, dst;\n\
    int4 sum0 = (int4)(0), sum1 = (int4)(0);\n\
    if(exclusive == 0 && rev)\n\
    {\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev == 0)\n\
    {\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxx, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        for(; coord.z < channel - 1;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.z++;\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(coord.z) * in_out_zp_scale + output_zp;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    else if(exclusive && rev)\n\
    {\n\
        coord.z = channel - 1;\n\
        int tmpAlpha0 = convert_int_rte(output_zp);\n\
        int4 tmpVal;\n\
        tmpVal.x = tmpAlpha0;\n\
        VXC_DP2x8(dst, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst.xxxxxxxx, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        for(; coord.z > 0;)\n\
        {\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP4x4(sum0, src, sum0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32A_4x4);\n\
            VXC_DP4x4(sum1, src, sum1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAccSumVertU8toI32B_4x4);\n\
            float tmpAlpha = convert_float(channel - coord.z) * in_out_zp_scale + output_zp;\n\
            coord.z--;\n\
            float4 tmpSum0 = convert_float4(sum0) * in_out_scale + tmpAlpha;\n\
            float4 tmpSum1 = convert_float4(sum1) * in_out_scale + tmpAlpha;\n\
            int4 tmpDst0 = convert_int4_rte(tmpSum0);\n\
            int4 tmpDst1 = convert_int4_rte(tmpSum1);\n\
            VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero,1),\n\
                        uniConvertInt32toUint8_2x8);\n\
\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
}\n\
"; /* end of cumsum_ex_rev_axis2_vx*/

static const char cumsum_f16_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccSumVertF16toF16_2x8;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16A_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16B_4x4;\n\
_viv_uniform VXC_512Bits uniSumHorzF16toF16C_2x8;\n\
_viv_uniform VXC_512Bits uniAccSumHorzF16toF16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSetZeroF16_2x8;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
#define CUMSUM_F16TOQINT_AXIS2(out_name, src_type, dst_type) \\\n\
__kernel void cumsum_F16to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CUMSUM_F16TOQINT_AXIS2(I8,  vxc_half8, vxc_char16)\n\
CUMSUM_F16TOQINT_AXIS2(I16, vxc_half8, vxc_short8)\n\
CUMSUM_F16TOQINT_AXIS2(U8,  vxc_half8, vxc_uchar16)\n\
\n\
\n\
#define CUMSUM_F16TOQINT_AXIS1(out_name, src_type, dst_type) \\\n\
__kernel void cumsum_F16to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CUMSUM_F16TOQINT_AXIS1(I8,  vxc_half8, vxc_char16)\n\
CUMSUM_F16TOQINT_AXIS1(I16, vxc_half8, vxc_short8)\n\
CUMSUM_F16TOQINT_AXIS1(U8,  vxc_half8, vxc_uchar16)\n\
\n\
#define CUMSUM_F16TOQINT_AXIS0(out_name, src_type, dst_type) \\\n\
__kernel void cumsum_F16to##out_name##_axis0( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, tmpsum, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16A_4x4); \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16B_4x4); \\\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumHorzF16toF16C_2x8); \\\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumHorzF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CUMSUM_F16TOQINT_AXIS0(I8,  vxc_half8, vxc_char16)\n\
CUMSUM_F16TOQINT_AXIS0(I16, vxc_half8, vxc_short8)\n\
CUMSUM_F16TOQINT_AXIS0(U8,  vxc_half8, vxc_uchar16)\n\
\n\
#define CUMSUM_F16TOQINT_AXIS1_2D(out_name, src_type, dst_type) \\\n\
__kernel void cumsum_F16to##out_name##_axis1_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    for(; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumVertF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CUMSUM_F16TOQINT_AXIS1_2D(I8,  vxc_half8, vxc_char16)\n\
CUMSUM_F16TOQINT_AXIS1_2D(I16, vxc_half8, vxc_short8)\n\
CUMSUM_F16TOQINT_AXIS1_2D(U8,  vxc_half8, vxc_uchar16)\n\
\n\
#define CUMSUM_F16TOQINT_AXIS0_2D(out_name, src_type, dst_type) \\\n\
__kernel void cumsum_F16to##out_name##_axis0_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, tmpsum, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    for(; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
 \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16A_4x4); \\\n\
        VXC_DP4x4(tmpsum, data, data, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16B_4x4); \\\n\
        VXC_DP2x8(tmpsum, tmpsum, tmpsum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSumHorzF16toF16C_2x8); \\\n\
        VXC_DP2x8(sum, tmpsum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAccSumHorzF16toF16_2x8); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
CUMSUM_F16TOQINT_AXIS0_2D(I8,  vxc_half8, vxc_char16)\n\
CUMSUM_F16TOQINT_AXIS0_2D(I16, vxc_half8, vxc_short8)\n\
CUMSUM_F16TOQINT_AXIS0_2D(U8,  vxc_half8, vxc_uchar16)\n\
\n\
#define CUMSUM_F16TOQINT_EX_REV_AXIS2(out_name, src_type, dst_type) \\\n\
__kernel void cumsum_ex_rev_F16to##out_name##_axis2( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.z < channel - 1;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.z++; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
     \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        coord.z = channel - 1; \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.z > 0;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.z--; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
     \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_F16TOQINT_EX_REV_AXIS2(I8,  vxc_half8, vxc_char16)\n\
CUMSUM_F16TOQINT_EX_REV_AXIS2(I16, vxc_half8, vxc_short8)\n\
CUMSUM_F16TOQINT_EX_REV_AXIS2(U8,  vxc_half8, vxc_uchar16)\n\
\n\
#define CUMSUM_F16TOQINT_EX_REV_AXIS1(out_name, src_type, dst_type) \\\n\
__kernel void cumsum_ex_rev_F16to##out_name##_axis1( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, int exclusive, int rev \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    dst_type dst; \\\n\
    vxc_half8 data, sum; \\\n\
    VXC_DP2x8(sum, sum, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSetZeroF16_2x8); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    if(exclusive == 0 && rev) \\\n\
    { \\\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev == 0) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.y < height - 1;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive && rev) \\\n\
    { \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        coord.y = height - 1; \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        for(; coord.y > 0;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y--; \\\n\
            _viv_asm(COPY, data, src, 16); \\\n\
            VXC_DP2x8(sum, data, sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccSumVertF16toF16_2x8); \\\n\
            VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_F16TOQINT_EX_REV_AXIS1(I8,  vxc_half8, vxc_char16)\n\
CUMSUM_F16TOQINT_EX_REV_AXIS1(I16, vxc_half8, vxc_short8)\n\
CUMSUM_F16TOQINT_EX_REV_AXIS1(U8,  vxc_half8, vxc_uchar16)\n\
"; /* end of cumsum_f16_u8_vx*/

static const char custom_letterbox_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int top;\n\
_viv_uniform int left;\n\
_viv_uniform float out_scale_r;\n\
_viv_uniform float out_scale_g;\n\
_viv_uniform float out_scale_b;\n\
_viv_uniform float out_zp_r;\n\
_viv_uniform float out_zp_g;\n\
_viv_uniform float out_zp_b;\n\
_viv_uniform float pad_v_r;\n\
_viv_uniform float pad_v_g;\n\
_viv_uniform float pad_v_b;\n\
_viv_uniform float scale_w;\n\
_viv_uniform float scale_h;\n\
_viv_uniform int resize_max_w;\n\
_viv_uniform int resize_max_h;\n\
_viv_uniform int out_height;\n\
_viv_uniform int r_order;\n\
_viv_uniform int b_order;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniLeftToFloat32_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
__kernel void custom_letterbox_U8toU8\n\
    (\n\
     __read_only  image2d_t input,\n\
     __write_only image2d_t output,\n\
            int       top_,\n\
            int       bottom_,\n\
            int       left_,\n\
            int       right_,\n\
            float     mean_r_,\n\
            float     mean_g_,\n\
            float     mean_b_,\n\
            float     scale_r_,\n\
            float     scale_g_,\n\
            float     scale_b_,\n\
            int       pad_r_,\n\
            int       pad_g_,\n\
            int       pad_b_,\n\
            int       reverse_channel\n\
    )\n\
{\n\
    int2 coord_out  =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord = coord_out;\n\
    uint4 dst = (uint4)(0,0,0,0);\n\
    vxc_uchar8 result;\n\
\n\
    if (coord_out.x < left || coord_out.x >= resize_max_w ||\n\
        coord_out.y < top  || coord_out.y >= resize_max_h)\n\
    {\n\
        dst.x = convert_uint(pad_v_r);\n\
        coord.y = coord_out.y + r_order;\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
        VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
        dst.x = convert_uint(pad_v_g);\n\
        coord.y = coord_out.y + out_height;\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
        VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
        dst.x = convert_uint(pad_v_b);\n\
        coord.y = coord_out.y + b_order;\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
        VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        return;\n\
    }\n\
\n\
    float in_x = convert_float(coord_out.x - left) * scale_w;\n\
    float in_y = convert_float(coord_out.y - top) * scale_h;\n\
    float left_x_f    = floor(in_x);\n\
    float top_y_f     = floor(in_y);\n\
    float x_lerp      = in_x - left_x_f;\n\
    float y_lerp      = in_y - top_y_f;\n\
    int   left_x_idx  = convert_int(left_x_f);\n\
    int   top_y_idx   = convert_int(top_y_f);\n\
\n\
    int2 coord_in = (int2)(3 * left_x_idx, top_y_idx);\n\
    vxc_uchar8 top_data, bottom_data;\n\
\n\
    VXC_ReadImage(top_data, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(bottom_data, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4 = (float4)(0,0,0,0);\n\
    float4 right4 = (float4)(0,0,0,0);\n\
    float4 top4 = (float4)(0,0,0,0);\n\
    float4 bottom4 = (float4)(0,0,0,0);\n\
    VXC_DP4x4(right4, top_data, top_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    VXC_DP4x4(left4, top_data, top_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniLeftToFloat32_4x4);\n\
    top4 = right4 * x_lerp + left4;\n\
    VXC_DP4x4(right4, bottom_data, bottom_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    VXC_DP4x4(left4, bottom_data, bottom_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniLeftToFloat32_4x4);\n\
    bottom4 = right4 * x_lerp + left4;\n\
    float4 out = (bottom4 - top4) * y_lerp + top4;\n\
\n\
    dst.x = convert_uint(out.x * out_scale_r + out_zp_r );\n\
    coord.y = coord_out.y + r_order;\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    dst.x = convert_uint(out.y * out_scale_g + out_zp_g);\n\
    coord.y = coord_out.y + out_height;\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    dst.x = convert_uint(out.z * out_scale_b + out_zp_b);\n\
    coord.y = coord_out.y + b_order;\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_letterbox_U8toI8\n\
    (\n\
     __read_only  image2d_t input,\n\
     __write_only image2d_t output,\n\
            int       top_,\n\
            int       bottom_,\n\
            int       left_,\n\
            int       right_,\n\
            float     mean_r_,\n\
            float     mean_g_,\n\
            float     mean_b_,\n\
            float     scale_r_,\n\
            float     scale_g_,\n\
            float     scale_b_,\n\
            int       pad_r_,\n\
            int       pad_g_,\n\
            int       pad_b_,\n\
            int       reverse_channel\n\
    )\n\
{\n\
    int2 coord_out  =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord = coord_out;\n\
    int4 dst = (int4)(0,0,0,0);\n\
    vxc_char8 result;\n\
\n\
    if (coord_out.x < left || coord_out.x >= resize_max_w ||\n\
        coord_out.y < top  || coord_out.y >= resize_max_h)\n\
    {\n\
        dst.x = convert_int(pad_v_r);\n\
        coord.y = coord_out.y + r_order;\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
        VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
        dst.x = convert_int(pad_v_g);\n\
        coord.y = coord_out.y + out_height;\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
        VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
        dst.x = convert_int(pad_v_b);\n\
        coord.y = coord_out.y + b_order;\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
        VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        return;\n\
    }\n\
\n\
    float in_x = convert_float(coord_out.x - left) * scale_w;\n\
    float in_y = convert_float(coord_out.y - top) * scale_h;\n\
    float left_x_f    = floor(in_x);\n\
    float top_y_f     = floor(in_y);\n\
    float x_lerp      = in_x - left_x_f;\n\
    float y_lerp      = in_y - top_y_f;\n\
    int   left_x_idx  = convert_int(left_x_f);\n\
    int   top_y_idx   = convert_int(top_y_f);\n\
\n\
    int2 coord_in = (int2)(3 * left_x_idx, top_y_idx);\n\
    vxc_char8 top_data, bottom_data;\n\
\n\
    VXC_ReadImage(top_data, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                               VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(bottom_data, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), \\\n\
                               VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4 = (float4)(0,0,0,0);\n\
    float4 right4 = (float4)(0,0,0,0);\n\
    float4 top4 = (float4)(0,0,0,0);\n\
    float4 bottom4 = (float4)(0,0,0,0);\n\
    VXC_DP4x4(right4, top_data, top_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    VXC_DP4x4(left4, top_data, top_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniLeftToFloat32_4x4);\n\
    top4 = right4 * x_lerp + left4;\n\
    VXC_DP4x4(right4, bottom_data, bottom_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    VXC_DP4x4(left4, bottom_data, bottom_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniLeftToFloat32_4x4);\n\
    bottom4 = right4 * x_lerp + left4;\n\
    float4 out = (bottom4 - top4) * y_lerp + top4;\n\
\n\
    dst.x = convert_int(out.x * out_scale_r + out_zp_r);\n\
    coord.y = coord_out.y + r_order;\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    dst.x = convert_int(out.y * out_scale_g + out_zp_g);\n\
    coord.y = coord_out.y + out_height;\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    dst.x = convert_int(out.z * out_scale_b + out_zp_b);\n\
    coord.y = coord_out.y + b_order;\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_letterbox_U8toF16\n\
    (\n\
     __read_only  image2d_t input,\n\
     __write_only image2d_t output,\n\
            int       top_,\n\
            int       bottom_,\n\
            int       left_,\n\
            int       right_,\n\
            float     mean_r_,\n\
            float     mean_g_,\n\
            float     mean_b_,\n\
            float     scale_r_,\n\
            float     scale_g_,\n\
            float     scale_b_,\n\
            int       pad_r_,\n\
            int       pad_g_,\n\
            int       pad_b_,\n\
            int       reverse_channel\n\
    )\n\
{\n\
    int2 coord_out  =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord = coord_out;\n\
    half4 tmp;\n\
    vxc_half8 dst_temp;\n\
    vxc_ushort8 dst;\n\
\n\
    if (coord_out.x < left || coord_out.x >= resize_max_w ||\n\
        coord_out.y < top  || coord_out.y >= resize_max_h)\n\
    {\n\
\n\
        float4 pad = (float4)(pad_v_r, pad_v_g, pad_v_b, 0);\n\
        _viv_asm(CONV, tmp, pad);\n\
        VXC_DP2x8(dst_temp, tmp, tmp, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, dst, dst_temp, 16);\n\
        coord.y = coord_out.y + r_order;\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
        tmp.x = tmp.y;\n\
        VXC_DP2x8(dst_temp, tmp, tmp, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, dst, dst_temp, 16);\n\
        coord.y = coord_out.y + out_height;\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
        tmp.x = tmp.z;\n\
        VXC_DP2x8(dst_temp, tmp, tmp, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, dst, dst_temp, 16);\n\
        coord.y = coord_out.y + b_order;\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        return;\n\
    }\n\
\n\
    float in_x = convert_float(coord_out.x - left) * scale_w;\n\
    float in_y = convert_float(coord_out.y - top) * scale_h;\n\
    float left_x_f    = floor(in_x);\n\
    float top_y_f     = floor(in_y);\n\
    float x_lerp      = in_x - left_x_f;\n\
    float y_lerp      = in_y - top_y_f;\n\
    int   left_x_idx  = convert_int(left_x_f);\n\
    int   top_y_idx   = convert_int(top_y_f);\n\
\n\
    int2 coord_in = (int2)(3 * left_x_idx, top_y_idx);\n\
    vxc_uchar8 top_data, bottom_data;\n\
\n\
    VXC_ReadImage(top_data, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(bottom_data, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), \\\n\
                            VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4 = (float4)(0,0,0,0);\n\
    float4 right4 = (float4)(0,0,0,0);\n\
    float4 top4 = (float4)(0,0,0,0);\n\
    float4 bottom4 = (float4)(0,0,0,0);\n\
    VXC_DP4x4(right4, top_data, top_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    VXC_DP4x4(left4, top_data, top_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniLeftToFloat32_4x4);\n\
    top4 = right4 * x_lerp + left4;\n\
    VXC_DP4x4(right4, bottom_data, bottom_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    VXC_DP4x4(left4, bottom_data, bottom_data, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniLeftToFloat32_4x4);\n\
    bottom4 = right4 * x_lerp + left4;\n\
    float4 out = (bottom4 - top4) * y_lerp + top4;\n\
\n\
    float4 out_temp = (float4)(0,0,0,0);\n\
    out_temp.x = out.x * out_scale_r + out_zp_r;\n\
    _viv_asm(CONV, tmp, out_temp);\n\
    VXC_DP2x8(dst_temp, tmp, tmp, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
    _viv_asm(COPY, dst, dst_temp, 16);\n\
    coord.y = coord_out.y + r_order;\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    out_temp.x = out.y * out_scale_g + out_zp_g;\n\
    _viv_asm(CONV, tmp, out_temp);\n\
    VXC_DP2x8(dst_temp, tmp, tmp, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
    _viv_asm(COPY, dst, dst_temp, 16);\n\
    coord.y = coord_out.y + out_height;\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    out_temp.x = out.z * out_scale_b + out_zp_b;\n\
    _viv_asm(CONV, tmp, out_temp);\n\
    VXC_DP2x8(dst_temp, tmp, tmp, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
    _viv_asm(COPY, dst, dst_temp, 16);\n\
    coord.y = coord_out.y + out_height;\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of custom_letterbox_vx*/

static const char custom_softmax_vx[] = "/*\n\
 ============================================================================\n\
 Name        : Softmax2.vx\n\
 Author      : VSI\n\
 Version     :\n\
 Copyright   : Your copyright notice\n\
 Description :\n\
 ============================================================================\n\
 */\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits Uni4x4_Fp16ToFp32;\n\
_viv_uniform VXC_512Bits uniExtract8Bin_2x8;\n\
_viv_uniform int  sf_size;\n\
_viv_uniform float  srcScale;\n\
_viv_uniform float  srcZP;\n\
_viv_uniform float  dstScale;\n\
_viv_uniform float  dstZP;\n\
 #define F_MAX(a,b) ((a)>(b)?(a):(b))\n\
__kernel void Softmax2VXC\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int axis\n\
    )\n\
{\n\
   int4 coord_in = (int4)(0, get_global_id(0), 0, 0);\n\
   float fMax = 0;\n\
   for (int i = 0; i < sf_size; i++)\n\
   {\n\
       vxc_short8 val;\n\
       vxc_half8  val_h;\n\
       coord_in.x = i;\n\
       VXC_ReadImage2DArray(val, input, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
       _viv_asm(COPY, val_h, val, 16);\n\
       float fval;\n\
       VXC_DP4x4(fval, val_h, val_h, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1), Uni4x4_Fp16ToFp32);\n\
\n\
       fMax = F_MAX(fMax, fval);\n\
   }\n\
    float  fProbSum = 0.0f;\n\
    vxc_short8 dst;\n\
    for (int i = 0; i < sf_size; i++)\n\
    {\n\
       vxc_short8 val;\n\
       vxc_half8  val_h;\n\
       coord_in.x = i;\n\
       VXC_ReadImage2DArray(val, input, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
       _viv_asm(COPY, val_h, val, 16);\n\
       float fval;\n\
       VXC_DP4x4(fval, val_h, val_h, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1), Uni4x4_Fp16ToFp32);\n\
       float fOut = (float)exp(fval - fMax);\n\
       fProbSum += fOut;\n\
       half hVal;\n\
       _viv_asm(CONV, hVal, fOut);\n\
       _viv_asm(COPY, dst, hVal, 4);\n\
\n\
       VXC_WriteImage2DArray(output, coord_in, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
\n\
    for (int i = 0; i < sf_size; i++)\n\
    {\n\
       vxc_short8 val;\n\
       vxc_half8  val_h;\n\
       coord_in.x = i;\n\
       VXC_ReadImage2DArray(val, output, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
       float fval;\n\
       _viv_asm(COPY, val_h,val, 16);\n\
       VXC_DP4x4(fval, val_h, val_h, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1), Uni4x4_Fp16ToFp32);\n\
       float fOut =fval / fProbSum;\n\
       half hVal;\n\
       _viv_asm(CONV, hVal, fOut);\n\
       _viv_asm(COPY,dst,hVal, 4);\n\
       VXC_WriteImage2DArray(output, coord_in, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void Softmax2VXC_u8\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int axis\n\
    )\n\
{\n\
   int4 coord_in = (int4)(0, get_global_id(0), 0, 0);\n\
   float fMax = -3.4e38f;\n\
   for (int i = 0; i < sf_size; i++)\n\
   {\n\
       vxc_uchar8 val;\n\
       coord_in.x = i;\n\
       VXC_ReadImage2DArray(val, input, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
       float fval;\n\
       VXC_DP4x4(fval, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1), Uni4x4_Fp16ToFp32);\n\
       fval = (fval - srcZP) * srcScale;\n\
       fMax = F_MAX(fMax, fval);\n\
   }\n\
\n\
    float  fProbSum = 0.0f;\n\
    vxc_uchar8 dst;\n\
    for (int i = 0; i < sf_size; i++)\n\
    {\n\
       vxc_uchar8 val;\n\
\n\
       coord_in.x = i;\n\
       VXC_ReadImage2DArray(val, input, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
       float fval;\n\
       VXC_DP4x4(fval, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1), Uni4x4_Fp16ToFp32);\n\
       fval = (fval - srcZP) * srcScale;\n\
       float fOut = (float)exp(fval - fMax);\n\
       fProbSum += fOut;\n\
    }\n\
\n\
    for (int i = 0; i < sf_size; i++)\n\
    {\n\
       vxc_uchar8 val;\n\
       coord_in.x = i;\n\
       VXC_ReadImage2DArray(val, input, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
       float fval;\n\
       VXC_DP4x4(fval, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1), Uni4x4_Fp16ToFp32);\n\
       fval = (fval - srcZP) * srcScale;\n\
\n\
       float fOut = exp(fval - fMax) / fProbSum;\n\
\n\
       fOut = fOut * dstScale + dstZP;\n\
       short dst0;\n\
       _viv_asm(CONV, dst0, fOut);\n\
       VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1), uniExtract8Bin_2x8);\n\
       VXC_WriteImage2DArray(output, coord_in, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of custom_softmax_vx*/

static const char custom_warp_affine_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float4 matrix0;\n\
_viv_uniform float2 matrix1;\n\
_viv_uniform float4 matrix4;\n\
\n\
__kernel void custom_warp_affine_nearest_neighbor_U8toU8\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_bilinear_U8toU8\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of custom_warp_affine_vx*/

static const char custom_warp_affine_2d_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float4 matrix0;\n\
_viv_uniform float2 matrix1;\n\
_viv_uniform float4 matrix4;\n\
__kernel void custom_warp_affine_nearest_neighbor_U8toU8_2D\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_bilinear_U8toU8_2D\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of custom_warp_affine_2d_vx*/

static const char custom_warp_affine_optional_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define WARP_AFFINE(name) \\\n\
__kernel void custom_warp_affine_##name \\\n\
( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       matrix, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           _m0, \\\n\
                 float           _m1, \\\n\
                 float           _m2, \\\n\
                 float           _m3, \\\n\
                 float           _m4, \\\n\
                 float           _m5 \\\n\
) \\\n\
\n\
#define GET_MATRIX_VALUE \\\n\
    float4 matrix0; \\\n\
    float2 matrix1; \\\n\
    float4 matrix4; \\\n\
    int2   coord_matrix = (int2)(0,0); \\\n\
    Image  img1 = create_image_from_image2d(matrix, 4); \\\n\
    __global float* matrix_ptr = (__global float*)img1.ptr; \\\n\
    matrix0 = vload4(0, matrix_ptr); \\\n\
    matrix1 = vload2(2, matrix_ptr); \\\n\
    matrix4.x = matrix0.x; \\\n\
    matrix4.y = matrix0.y; \\\n\
    matrix4.z = matrix0.x * 2; \\\n\
    matrix4.w = matrix0.y * 2; \\\n\
\n\
WARP_AFFINE(nearest_neighbor_U8toU8_2D_optional_input)\n\
{\n\
    int2   coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
WARP_AFFINE(bilinear_U8toU8_2D_optional_input)\n\
{\n\
    int2   coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
WARP_AFFINE(nearest_neighbor_U8toU8_optional_input)\n\
{\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
WARP_AFFINE(bilinear_U8toU8_optional_input)\n\
{\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f = coord_f.zwzw + matrix4;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of custom_warp_affine_optional_vx*/

static const char custom_warp_affine_rgb_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float4 matrix0;\n\
_viv_uniform float2 matrix1;\n\
__kernel void custom_warp_affine_nearest_neighbor_U8toU8_rgb_2D\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0) * 3, get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = coord_in.x + 1;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = coord_in.x + 1;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z = coord_in.z + 1;\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z = coord_in.z + 1;\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_bilinear_U8toU8_rgb_2D\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0) * 3, get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    vxc_uchar16 src0, src1, src_0, src_1, dst;\n\
    VXC_ReadImage(src_0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src_1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src_0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src_1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_nearest_neighbor_U8toU8_rgb\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0) * 3, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_bilinear_U8toU8_rgb\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0) * 3, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 src0, src1, src_0, src_1, dst;\n\
    VXC_OP4(img_load_3d, src_0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src_1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src_0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src_1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of custom_warp_affine_rgb_vx*/

static const char custom_warp_affine_rgb_optional_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define GET_MATRIX_VALUE \\\n\
    float4 matrix0; \\\n\
    float2 matrix1; \\\n\
    Image  img1 = create_image_from_image2d(matrix, 4); \\\n\
    __global float* matrix_ptr = (__global float*)img1.ptr; \\\n\
    matrix0 = vload4(0, matrix_ptr); \\\n\
    matrix1 = vload2(2, matrix_ptr); \\\n\
\n\
__kernel void custom_warp_affine_nearest_neighbor_U8toU8_rgb_2D_optional_input\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       matrix,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0) * 3, get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
    int2   coord_matrix = (int2)(0,0);\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = coord_in.x + 1;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = coord_in.x + 1;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z = coord_in.z + 1;\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z = coord_in.z + 1;\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_bilinear_U8toU8_rgb_2D_optional_input\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       matrix,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0) * 3, get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
    int2   coord_matrix = (int2)(0,0);\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    vxc_uchar16 src0, src1, src_0, src_1, dst;\n\
    VXC_ReadImage(src_0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src_1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src_0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src_1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_nearest_neighbor_U8toU8_rgb_optional_input\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       matrix,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0) * 3, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
    int2   coord_matrix = (int2)(0,0);\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    coord_input.x = coord_input.x + 1;\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_affine_bilinear_U8toU8_rgb_optional_input\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       matrix,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0) * 3, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f = convert_float4(coord_in);\n\
    int2   coord_matrix = (int2)(0,0);\n\
    GET_MATRIX_VALUE\n\
\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy;\n\
\n\
    coord_in.x = floor(coord_f.x) * 3;\n\
    coord_in.y = floor(coord_f.y);\n\
    coord_in.z = floor(coord_f.z) * 3;\n\
    coord_in.w = floor(coord_f.w);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 src0, src1, src_0, src_1, dst;\n\
    VXC_OP4(img_load_3d, src_0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src_1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    VXC_OP4(img_load_3d, src_0, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src_1, input, coord_input.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    src0.x = src_0.s0;\n\
    src0.y = src_0.s3;\n\
    src1.x = src_1.s0;\n\
    src1.y = src_1.s3;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s1;\n\
    src0.y = src_0.s4;\n\
    src1.x = src_1.s1;\n\
    src1.y = src_1.s4;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    src0.x = src_0.s2;\n\
    src0.y = src_0.s5;\n\
    src1.x = src_1.s2;\n\
    src1.y = src_1.s5;\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of custom_warp_affine_rgb_optional_vx*/

static const char custom_warp_perspective_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float4 matrix0;\n\
_viv_uniform float4 matrix1;\n\
_viv_uniform float4 matrix2;\n\
_viv_uniform float4 matrix4;\n\
__kernel void custom_warp_perspective_nearest_neighbor_U8toU8_2D\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5,\n\
                 float           _m6,\n\
                 float           _m7,\n\
                 float           _m8\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f0 = convert_float4(coord_in);\n\
\n\
    float4 z0 = coord_f0.xzxz * matrix1.zzzz + coord_f0.y * matrix1.wwww + matrix2.xxxx;\n\
    z0.zw = z0.zw + 2 * matrix1.z;\n\
    float4 z1 = z0 + 4 * matrix1.z;\n\
\n\
    z0 = 1.0f / z0;\n\
    z1 = 1.0f / z1;\n\
\n\
    coord_f0 = coord_f0.xxzz * matrix0.xyxy + coord_f0.yyww * matrix0.zwzw + matrix1.xyxy;\n\
    float4 coord_f = coord_f0 * z0.xxyy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    vxc_uchar16 dst;\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z0.zzww;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.xxyy;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.zzww;\n\
    coord_in = convert_int4(coord_f);\n\
    VXC_ReadImage(dst, input, coord_in.xy, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(dst, input, coord_in.zw, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_perspective_bilinear_U8toU8_2D\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5,\n\
                 float           _m6,\n\
                 float           _m7,\n\
                 float           _m8\n\
)\n\
{\n\
    int2   coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f0 = convert_float4(coord_in);\n\
\n\
    float4 z0 = coord_f0.xzxz * matrix1.zzzz + coord_f0.y * matrix1.wwww + matrix2.xxxx;\n\
    z0.zw = z0.zw + 2 * matrix1.z;\n\
    float4 z1 = z0 + 4 * matrix1.z;\n\
\n\
    z0 = 1.0f / z0;\n\
    z1 = 1.0f / z1;\n\
\n\
    coord_f0 = coord_f0.xxzz * matrix0.xyxy + coord_f0.yyww * matrix0.zwzw + matrix1.xyxy;\n\
    float4 coord_f = coord_f0 * z0.xxyy;\n\
\n\
    coord_in = convert_int4(floor(coord_f));\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z0.zzww;\n\
    coord_in = convert_int4(floor(coord_f));\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.xxyy;\n\
    coord_in = convert_int4(floor(coord_f));\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.zzww;\n\
    coord_in = convert_int4(floor(coord_f));\n\
    VXC_ReadImage(src0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_ReadImage(src0, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in.zw, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define IMAGE_LOAD_3D(dst, xoffset, yoffset, start, end) \\\n\
    VXC_OP4(img_load_3d, dst, input, coord_input.xywz, VXC_5BITOFFSET_XY(xoffset, yoffset), \\\n\
        VXC_MODIFIER(start, end, 0, VXC_RM_TowardZero, 0));\n\
__kernel void custom_warp_perspective_nearest_neighbor_U8toU8\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5,\n\
                 float           _m6,\n\
                 float           _m7,\n\
                 float           _m8\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f0 = convert_float4(coord_in);\n\
\n\
    float4 z0 = coord_f0.xzxz * matrix1.zzzz + coord_f0.y * matrix1.wwww + matrix2.xxxx;\n\
    z0.zw = z0.zw + 2 * matrix1.z;\n\
    float4 z1 = z0 + 4 * matrix1.z;\n\
\n\
    z0 = 1.0f / z0;\n\
    z1 = 1.0f / z1;\n\
\n\
    coord_f0 = coord_f0.xxzz * matrix0.xyxy + coord_f0.yyww * matrix0.zwzw + matrix1.xyxy;\n\
    float4 coord_f = coord_f0 * z0.xxyy;\n\
\n\
    coord_in = convert_int4(coord_f);\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 dst;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 0, 0)\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 1, 1)\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z0.zzww;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 2, 2)\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 3, 3)\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.xxyy;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 4, 4)\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 5, 5)\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.zzww;\n\
    coord_in = convert_int4(coord_f);\n\
    coord_input.xy = coord_in.xy;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 6, 6)\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(dst, 0, 0, 7, 7)\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void custom_warp_perspective_bilinear_U8toU8\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           _m0,\n\
                 float           _m1,\n\
                 float           _m2,\n\
                 float           _m3,\n\
                 float           _m4,\n\
                 float           _m5,\n\
                 float           _m6,\n\
                 float           _m7,\n\
                 float           _m8\n\
)\n\
{\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1));\n\
\n\
    float4 coord_f0 = convert_float4(coord_in);\n\
\n\
    float4 z0 = coord_f0.xzxz * matrix1.zzzz + coord_f0.y * matrix1.wwww + matrix2.xxxx;\n\
    z0.zw = z0.zw + 2 * matrix1.z;\n\
    float4 z1 = z0 + 4 * matrix1.z;\n\
\n\
    z0 = 1.0f / z0;\n\
    z1 = 1.0f / z1;\n\
\n\
    coord_f0 = coord_f0.xxzz * matrix0.xyxy + coord_f0.yyww * matrix0.zwzw + matrix1.xyxy;\n\
    float4 coord_f = coord_f0 * z0.xxyy;\n\
\n\
    coord_in = convert_int4(floor(coord_f));\n\
\n\
    int4 coord_input = (int4)(coord_in.xy, get_global_id(2), get_global_id(2));\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_input.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_input.w, baseAddr);\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z0.zzww;\n\
    coord_in = convert_int4(floor(coord_f));\n\
    coord_input.xy = coord_in.xy;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.xxyy;\n\
    coord_in = convert_int4(floor(coord_f));\n\
    coord_input.xy = coord_in.xy;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_f0 = coord_f0.zwzw + matrix4;\n\
    coord_f = coord_f0 * z1.zzww;\n\
    coord_in = convert_int4(floor(coord_f));\n\
    coord_input.xy = coord_in.xy;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.xy, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.y, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.x, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    coord_input.xy = coord_in.zw;\n\
    IMAGE_LOAD_3D(src0, 0, 0, 0, 1)\n\
    IMAGE_LOAD_3D(src1, 0, 1, 0, 1)\n\
#if (VX_VERSION==1)\n\
    VXC_BiLinear(dst, src0, src1, coord_f.zw, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#else\n\
    VXC_Lerp(src0, src0, src1, coord_f.w, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    src1.s0 = src0.s1;\n\
    VXC_Lerp(dst, src0, src1, coord_f.z, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
#endif\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of custom_warp_perspective_vx*/

static const char depth2space_crd_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_ExLo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_ExHi_2x8;\n\
_viv_uniform VXC_512Bits uniDepth2SpaceF16Blk2_lo_2x8;\n\
_viv_uniform VXC_512Bits uniDepth2SpaceF16Blk2_hi_2x8;\n\
\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_QINT(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##to##src1_type_name( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int block_e2 = block_size * block_size; \\\n\
    ushort blk = (ushort)block_size; \\\n\
    int inx = (int)((ushort)gidx / blk); \\\n\
    int iny = (int)((ushort)gidy / blk); \\\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2; \\\n\
    int4 coord_in = (int4)(inx, iny, inz, 0); \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(dst,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_QINT(U8, U8, vxc_uchar16, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_QINT(I8, I8, vxc_char16, vxc_char16)\n\
DEPTH2SPACE_CRD_QINT_TO_QINT(I16, I16, vxc_short8, vxc_short8)\n\
\n\
__kernel void depth2space_crd_F16toF16(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int block_e2 = block_size * block_size;\n\
    ushort blk = (ushort)block_size;\n\
    int inx = (int)((ushort)gidx / blk);\n\
    int iny = (int)((ushort)gidy / blk);\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2;\n\
    int4 coord_in = (int4)(inx, iny, inz, 0);\n\
    vxc_short8 data;\n\
    VXC_ReadImage2DArray(data,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord_out, data, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_F16(src0_type_name, read_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##toF16( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int block_e2 = block_size * block_size; \\\n\
    ushort blk = (ushort)block_size; \\\n\
    int inx = (int)((ushort)gidx / blk); \\\n\
    int iny = (int)((ushort)gidy / blk); \\\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2; \\\n\
    int4 coord_in = (int4)(inx, iny, inz, 0); \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  tmpDst; \\\n\
    vxc_short8  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(tmpDst,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst, tmpDst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_F16(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_F16(I8, vxc_char16)\n\
DEPTH2SPACE_CRD_QINT_TO_F16(I16, vxc_short8)\n\
\n\
#define DEPTH2SPACE_CRD_F16_TO_QINT(src1_type_name, write_type) \\\n\
__kernel void depth2space_crd_F16to##src1_type_name( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int block_e2 = block_size * block_size; \\\n\
    ushort blk = (ushort)block_size; \\\n\
    int inx = (int)((ushort)gidx / blk); \\\n\
    int iny = (int)((ushort)gidy / blk); \\\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2; \\\n\
    int4 coord_in = (int4)(inx, iny, inz, 0); \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage2DArray(src,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_half8 data; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_F16_TO_QINT(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_F16_TO_QINT(I8, vxc_char16)\n\
DEPTH2SPACE_CRD_F16_TO_QINT(I16, vxc_short8)\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_QINT_BLK2(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##to##src1_type_name##_blk2( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz); \\\n\
    int4 coord_in = coord_out >> 1; \\\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4; \\\n\
    coord_in.w = coord_in.z + 1; \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyzz, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyww, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExLo_2x8); \\\n\
    VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExHi_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_QINT_BLK2(U8, U8, vxc_uchar16, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_QINT_BLK2(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void depth2space_crd_F16toF16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 data0, data1, dst0, dst1;\n\
    VXC_ReadImage2DArray(data0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(data1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(dst0, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(dst1, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void depth2space_crd_I16toI16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 src0, src1, data0, data1, dst0, dst1;\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(dst0, data0, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    VXC_DP2x8(dst1, data1, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_F16_BLK2(src0_type_name, read_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##toF16_blk2( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz); \\\n\
    int4 coord_in = coord_out >> 1; \\\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4; \\\n\
    coord_in.w = coord_in.z + 1; \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyzz, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyww, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  tmpDst0, tmpDst1; \\\n\
    vxc_short8  dst0, dst1; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(tmpDst0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExLo_2x8); \\\n\
    VXC_DP2x8(tmpDst1, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExHi_2x8); \\\n\
    _viv_asm(COPY, dst0, tmpDst0, 16); \\\n\
    _viv_asm(COPY, dst1, tmpDst1, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_out.x+=8; \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_F16_BLK2(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_F16_BLK2(I8, vxc_char16)\n\
\n\
__kernel void depth2space_crd_I16toF16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 src0, src1, data0, data1, dst0, dst1;\n\
    vxc_half8 tmpDst0, tmpDst1;\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(tmpDst0, data0, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    VXC_DP2x8(tmpDst1, data1, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    _viv_asm(COPY, dst0, tmpDst0, 16);\n\
    _viv_asm(COPY, dst1, tmpDst1, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define DEPTH2SPACE_CRD_F16_TO_QINT_BLK2(src1_type_name, write_type) \\\n\
__kernel void depth2space_crd_F16to##src1_type_name##_blk2( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz); \\\n\
    int4 coord_in = coord_out >> 1; \\\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4; \\\n\
    coord_in.w = coord_in.z + 1; \\\n\
    vxc_short8 src0, src1, data0, data1; \\\n\
    vxc_half8 tmpDst0, tmpDst1; \\\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz, \\\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww, \\\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8); \\\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, tmpDst0, data0, 16); \\\n\
    _viv_asm(COPY, tmpDst1, data1, 16); \\\n\
    VXC_DP2x8(dst, tmpDst0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_DP2x8(dst, tmpDst1, ms0, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_F16_TO_QINT_BLK2(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_F16_TO_QINT_BLK2(I8, vxc_char16)\n\
\n\
__kernel void depth2space_crd_F16toI16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 src0, src1, data0, data1, dst0, dst1;\n\
    vxc_half8 tmpDst0, tmpDst1;\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
    _viv_asm(COPY, tmpDst0, data0, 16);\n\
    _viv_asm(COPY, tmpDst1, data1, 16);\n\
    VXC_DP2x8(dst0, tmpDst0, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    VXC_DP2x8(dst1, tmpDst1, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of depth2space_crd_vx*/

static const char depthwise_conv1d_src0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_align8_step0_16x1;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_align8_step1_16x1;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
_viv_uniform int kernel_size_x16;\n\
_viv_uniform int kernel_size_x8;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_KN_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int2 coord_in = (int2)(get_global_id(0) * stride - pad, get_global_id(1));\n\
int4 coord = (int4)(get_global_id(0), 0, 0, get_global_id(1));\n\
vxc_uchar4 zp;\n\
vxc_uchar16 src;\n\
vxc_uchar16 w;\n\
vxc_short8 coef;\n\
int4 sum, sum0;\n\
\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
sum = read_imagei(bias, coord.wz);\n\
\n\
while(coord.y < kernel_size_x16)\n\
{\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(w, weight, coord.yw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step0_16x1);\n\
    sum.x += sum0.x;\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step1_16x1);\n\
    sum.x += sum0.x;\n\
    coord_in.x += 16;\n\
    coord.y += 16;\n\
}\n\
\n\
if (kernel_size_x8)\n\
{\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(w, weight, coord.yw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step0_16x1);\n\
    sum.x += sum0.x;\n\
}\n\
float4 result = convert_float4(sum.x) * scale + outputZP;\n\
uchar4 dst = convert_uchar4_sat(result);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_KN_D2(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int2 coord_in = (int2)(get_global_id(0) * stride - pad, get_global_id(1));\n\
int4 coord = (int4)(get_global_id(0), 0, 0, get_global_id(1));\n\
vxc_uchar4 zp;\n\
vxc_uchar16 src;\n\
vxc_uchar16 w;\n\
vxc_short8 coef;\n\
int4 sum, sum0;\n\
\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
sum = read_imagei(bias, coord.wz);\n\
\n\
while(coord.y < kernel_size_x8)\n\
{\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(w, weight, coord.yw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step0_16x1);\n\
    sum.x += sum0.x;\n\
    coord_in.x += 16;\n\
    coord.y += 8;\n\
}\n\
\n\
float4 result = convert_float4(sum.x) * scale + outputZP;\n\
uchar4 dst = convert_uchar4_sat(result);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of depthwise_conv1d_src0_vx*/

static const char depthwise_conv1d_src1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe0_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe1_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe2_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe3_8x2b;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe4_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe5_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe6_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe7_8x2b;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K40_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
vxc_uchar32 src0, src1;\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w0, w1, w2;\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src0.hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src0.lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src1.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(s2, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
int4 coord = (int4)(get_global_id(0), 16, 32, get_global_id(1));\n\
VXC_ReadImage(w0, weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w1, weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w2, weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w0, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w0, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
src1.hi = src0.lo;\n\
VXC_DP2x8(coef, w1, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w1, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
src0.hi = src1.lo;\n\
src0.lo = s2;\n\
VXC_DP2x8(coef, w2, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K56_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
vxc_uchar32 src0, src1, src2;\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w[4];\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src0.hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src0.lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src1.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src1.hi = src0.lo;\n\
VXC_ReadImage(src2.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src2.hi = src1.lo;\n\
int4 coord = (int4)(get_global_id(0), 16, 48, get_global_id(1));\n\
VXC_ReadImage(w[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[2], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[3], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of depthwise_conv1d_src1_vx*/

static const char depthwise_conv1d_src2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe0_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe1_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe2_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe3_8x2b;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe4_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe5_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe6_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe7_8x2b;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K64_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
int              pad,\n\
int              stride,\n\
int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
vxc_uchar32 src0, src1, src2, src3;\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w[4];\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src0.hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src0.lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src1.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src1.hi = src0.lo;\n\
VXC_ReadImage(src2.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src2.hi = src1.lo;\n\
coord_in.y += 16;\n\
VXC_ReadImage(src3.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0), \\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src3.hi = src2.lo;\n\
int4 coord = (int4)(get_global_id(0), 16, 48, get_global_id(1));\n\
VXC_ReadImage(w[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[2], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[3], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K80_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
int              pad,\n\
int              stride,\n\
int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 80, get_global_id(1), 0);\n\
vxc_uchar32 src[5];\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w[5];\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src[0].hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src[0].lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord_in.x += 32;\n\
VXC_ReadImage(src[1].lo, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[1].hi = src[0].lo;\n\
VXC_ReadImage(src[2].lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[2].hi = src[1].lo;\n\
VXC_ReadImage(src[3].lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[3].hi = src[2].lo;\n\
VXC_ReadImage(src[4].lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[4].hi = src[3].lo;\n\
int4 coord = (int4)(get_global_id(0), 16, 64, get_global_id(1));\n\
VXC_ReadImage(w[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord.y += 16;\n\
VXC_ReadImage(w[2], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[3], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[4], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of depthwise_conv1d_src2_vx*/

static const char depthwise_conv1d_src3_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe0_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe1_8x2b;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe4_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe5_8x2b;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K88_D2(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
int              pad,\n\
int              stride,\n\
int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
\n\
vxc_uchar32 src0, src1;\n\
vxc_uchar16 inData[12];\n\
vxc_uchar16 wData[6];\n\
int4 sumA, sumB;\n\
\n\
sumA = read_imagei(bias, coord_in.zw);\n\
\n\
VXC_ReadImage(inData[0], input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[1], input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[2], input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[3], input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord_in.xy += 64;\n\
VXC_ReadImage(inData[4], input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[5], input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[6], input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[7], input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord_in.xy += 64;\n\
VXC_ReadImage(inData[8], input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[9], input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[10], input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[11], input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
int4 coord = (int4)(get_global_id(0), 16, 48, get_global_id(1));\n\
\n\
VXC_ReadImage(wData[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[2], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[3], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord.yz += 64;\n\
VXC_ReadImage(wData[4], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[5], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
sumA = sumA.xxxx;\n\
sumB = sumA;\n\
\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
\n\
vxc_short8 coef;\n\
src0.hi = inData[0];\n\
src0.lo = inData[1];\n\
\n\
VXC_DP2x8(coef, wData[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = src0.lo;\n\
src0.lo = inData[2];\n\
\n\
VXC_DP2x8(coef, wData[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[2];\n\
src0.lo = inData[3];\n\
VXC_DP2x8(coef, wData[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = src0.lo;\n\
src0.lo = inData[4];\n\
\n\
VXC_DP2x8(coef, wData[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[4];\n\
src0.lo = inData[5];\n\
\n\
VXC_DP2x8(coef, wData[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = src0.lo;\n\
src0.lo = inData[6];\n\
\n\
VXC_DP2x8(coef, wData[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[6];\n\
src0.lo = inData[7];\n\
VXC_DP2x8(coef, wData[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = src0.lo;\n\
src0.lo = inData[8];\n\
VXC_DP2x8(coef, wData[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = inData[8];\n\
src0.lo = inData[9];\n\
VXC_DP2x8(coef, wData[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = src0.lo;\n\
src0.lo = inData[10];\n\
VXC_DP2x8(coef, wData[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[10];\n\
src0.lo = inData[11];\n\
VXC_DP2x8(coef, wData[5], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
float4 result0 = convert_float4(sumA) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of depthwise_conv1d_src3_vx*/

static const char detect_post_box_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform VXC_512Bits uniDataMerge_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZptoF32Conv0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZptoF32Conv1_4x4;\n\
_viv_uniform float logE;\n\
_viv_uniform int   input0_ZP;\n\
_viv_uniform int   input1_ZP;\n\
\n\
float exp_(float x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
__kernel void detect_post_box_F32_F32toF32(\n\
     __read_only image2d_array_t   input0,\n\
           __read_only image2d_t   input1,\n\
    __write_only image2d_array_t   output,\n\
                           float   inv_scale_y,\n\
                           float   inv_scale_x,\n\
                           float   inv_scale_h,\n\
                           float   inv_scale_w)\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    float4 dst;\n\
    float4 tmp0, tmp1, tmp2, tmp3;\n\
    uint4  tmp5, tmp6, tmp7;\n\
    src0 = read_imagef(input0, coord);\n\
    src1 = read_imagef(input1, coord.xy);\n\
    tmp0.x  = src1.x + src1.z * src0.x * inv_scale_y;\n\
    tmp0.y  = src1.y + src1.w * src0.y * inv_scale_x;\n\
    tmp1.x = src1.z * exp_(src0.z * inv_scale_h) * 0.5f;\n\
    tmp1.y = src1.w * exp_(src0.w * inv_scale_w) * 0.5f;\n\
    tmp2   = tmp0 - tmp1;\n\
    tmp3   = tmp0 + tmp1;\n\
    _viv_asm(COPY, tmp5, tmp2, 16);\n\
    _viv_asm(COPY, tmp6, tmp3, 16);\n\
    VXC_DP4x4(tmp7, tmp5, tmp6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDataMerge_4x4);\n\
    _viv_asm(COPY, dst, tmp7, 16);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void detect_post_box_U8_U8toF32(\n\
     __read_only image2d_array_t   input0,\n\
     __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t   output,\n\
                           float   inv_scale_y,\n\
                           float   inv_scale_x,\n\
                           float   inv_scale_h,\n\
                           float   inv_scale_w)\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    float4 dst;\n\
    float4 tmp0, tmp1, tmp2, tmp3;\n\
    vxc_uchar8 in0 = 0, in1 = 0;\n\
    vxc_short8 zp0 = (short)input0_ZP;\n\
    vxc_short8 zp1 = (short)input1_ZP;\n\
    VXC_ReadImage2DArray(in0, input0, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(src0, in0, zp0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniU8SubZptoF32Conv0_4x4);\n\
    VXC_DP4x4(src1, in1, zp1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniU8SubZptoF32Conv1_4x4);\n\
    tmp0.x  = src1.x + src1.z * src0.x * inv_scale_y;\n\
    tmp0.y  = src1.y + src1.w * src0.y * inv_scale_x;\n\
    tmp1.x = src1.z * exp_(src0.z * inv_scale_h) * 0.5f;\n\
    tmp1.y = src1.w * exp_(src0.w * inv_scale_w) * 0.5f;\n\
    dst.xy   = tmp0.xy - tmp1.xy;\n\
    dst.zw   = tmp0.xy + tmp1.xy;\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of detect_post_box_vx*/

static const char eltwise_unary_2d_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float alpha;\n\
_viv_uniform float beta;\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
#define rlogE    (0.693147182f)\n\
\n\
float4 eltwise_unary_hard_sigmoid(float4 x)\n\
{\n\
    x = alpha * x + beta;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
float4 _softrelu(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    x += 1;\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 _tanh(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return (2 * x - 1);\n\
}\n\
\n\
float4 eltwise_unary_mish(float4 x)\n\
{\n\
    float4 y = _softrelu(x);\n\
    x = x * _tanh(y);\n\
    return x;\n\
}\n\
\n\
float4 eltwise_unary_round(float4 x)\n\
{\n\
    return convert_float4(convert_int4_rte(x));\n\
}\n\
\n\
float4 evaluate_polynomial_alpha(float4 x2)\n\
{\n\
    float4 alpha0 = (float4){-2.72614225801306e-10f, 2.77068142495902e-08f,\n\
                            -2.10102402082508e-06f, -5.69250639462346e-05f};\n\
    float4 alpha1 = (float4){-7.34990630326855e-04f, -2.95459980854025e-03f,\n\
                            -1.60960333262415e-02f, 0};\n\
\n\
    float4 poly = alpha0.x * x2 + alpha0.y;\n\
    poly = poly * x2 + alpha0.z;\n\
    poly = poly * x2 + alpha0.w;\n\
    poly = poly * x2 + alpha1.x;\n\
    poly = poly * x2 + alpha1.y;\n\
    poly = poly * x2 + alpha1.z;\n\
\n\
    return poly;\n\
}\n\
\n\
float4 evaluate_polynomial_beta(float4 x2)\n\
{\n\
    float4 beta0 = (float4){-1.45660718464996e-05f, -2.13374055278905e-04f,\n\
                            -1.68282697438203e-03f, -7.37332916720468e-03f};\n\
    float4 beta1 = (float4){-1.42647390514189e-02f, 0, 0, 0};\n\
\n\
    float4 poly = beta0.x * x2 + beta0.y;\n\
    poly = poly * x2 + beta0.z;\n\
    poly = poly * x2 + beta0.w;\n\
    poly = poly * x2 + beta1.x;\n\
\n\
    return 1.0f / poly;\n\
}\n\
\n\
float4 erf_eval(float4 _x)\n\
{\n\
    float4 x = clamp(_x, -4, 4);\n\
    float4 x2 = x * x;\n\
\n\
    return x * evaluate_polynomial_alpha(x2) * evaluate_polynomial_beta(x2);\n\
}\n\
\n\
#define RSQRT2      (0.70710678118654752440084436210485f)\n\
float4 eltwise_unary_gelu(float4 x)\n\
{\n\
    float4 erf, data;\n\
    data = x * RSQRT2;\n\
    erf = erf_eval(data);\n\
    x = 0.5f * x * (1 + erf);\n\
\n\
    return x;\n\
}\n\
\n\
#define SQRT_2_RCP_PI  0.7978845834732056f\n\
float4 eltwise_unary_hard_gelu(float4 x)\n\
{\n\
    float4 cdf = 0.5f + 0.5f * _tanh(SQRT_2_RCP_PI *\n\
                        (x + 0.044715f * x * x * x));\n\
    return x * cdf;\n\
}\n\
\n\
float4 eltwise_unary_inverse_sigmoid(float4 x)\n\
{\n\
    float4 x1, x2;\n\
    x = clamp(x, 0, 1);\n\
    x1 = clamp(x, alpha, 1);\n\
    x2 = clamp((1 - x), alpha, 1);\n\
    return log(x1 / x2);\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define ELTSISE_UNARY_2D(func_name, src_type_name, dst_type_name, src_type, \\\n\
        src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
    __kernel void func_name##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecB = vecB * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
    vecB = vecB * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//MISH\n\
ELTSISE_UNARY_2D(mish, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(mish, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(mish, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(mish, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(mish, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(mish, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(mish, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(mish, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(mish, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(mish, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//ROUND\n\
ELTSISE_UNARY_2D(round, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(round, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(round, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(round, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(round, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(round, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(round, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(round, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(round, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(round, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//GELU\n\
ELTSISE_UNARY_2D(gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_GELU\n\
ELTSISE_UNARY_2D(hard_gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//INVERSE_SIGMOID\n\
ELTSISE_UNARY_2D(inverse_sigmoid, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(inverse_sigmoid, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define ELTSISE_UNARY_BF16_2D(func_name) \\\n\
    __kernel void func_name##_BF16toBF16_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//MISH\n\
ELTSISE_UNARY_BF16_2D(mish)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_BF16_2D(hard_sigmoid)\n\
//ROUND\n\
ELTSISE_UNARY_BF16_2D(round)\n\
//GELU\n\
ELTSISE_UNARY_BF16_2D(gelu)\n\
//HARD_GELU\n\
ELTSISE_UNARY_BF16_2D(hard_gelu)\n\
//INVERSE_SIGMOID\n\
ELTSISE_UNARY_BF16_2D(inverse_sigmoid)"; /* end of eltwise_unary_2d_0_vx*/

static const char eltwise_unary_2d_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float alpha;\n\
_viv_uniform float beta;\n\
\n\
float4 eltwise_unary_sin(float4 x)\n\
{\n\
    return native_sin(x);\n\
}\n\
\n\
float4 eltwise_unary_cos(float4 x)\n\
{\n\
    return native_cos(x);\n\
}\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
float4 eltwise_unary_exp(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
#define rlogE    (0.693147182f)\n\
float4 eltwise_unary_log(float4 x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 eltwise_unary_neg(float4 x)\n\
{\n\
    return x * -1;\n\
}\n\
\n\
float4 eltwise_unary_selu(float4 val)\n\
{\n\
    float4 x = val * logE;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val * beta;\n\
}\n\
\n\
float4 eltwise_unary_celu(float4 val)\n\
{\n\
    float4 x = val * logE * beta;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val;\n\
}\n\
\n\
float4 eltwise_unary_rcp(float4 val)\n\
{\n\
    return 1.0f / val;\n\
}\n\
\n\
float4 eltwise_unary_sign(float4 val)\n\
{\n\
    return sign(val);\n\
}\n\
\n\
float4 eltwise_unary_softsign(float4 val)\n\
{\n\
    float4 _rcp = 1.0f / (1.0f + fabs(val));\n\
    return val * _rcp;\n\
}\n\
\n\
float4 eltwise_unary_atan(float4 val)\n\
{\n\
    return atan(val);\n\
}\n\
\n\
float4 eltwise_unary_atanh(float4 val)\n\
{\n\
    return atanh(val);\n\
}\n\
\n\
float4 eltwise_unary_acosh(float4 val)\n\
{\n\
    return acosh(val);\n\
}\n\
\n\
float4 eltwise_unary_tan(float4 val)\n\
{\n\
    return native_tan(val);\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define ELTSISE_UNARY_2D(func_name, src_type_name, dst_type_name, src_type, \\\n\
        src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
    __kernel void func_name##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecB = vecB * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
    vecB = vecB * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define ELTSISE_UNARY_BF16_2D(func_name) \\\n\
    __kernel void func_name##_BF16toBF16_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
#define ADD_ELTSISE_UNARY_2D(func_name) \\\n\
ELTSISE_UNARY_2D(func_name, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_2D(func_name, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8) \\\n\
ELTSISE_UNARY_2D(func_name, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8) \\\n\
ELTSISE_UNARY_2D(func_name, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8) \\\n\
ELTSISE_UNARY_2D(func_name, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8) \\\n\
ELTSISE_UNARY_2D(func_name, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_2D(func_name, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8) \\\n\
ELTSISE_UNARY_2D(func_name, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_2D(func_name, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8) \\\n\
ELTSISE_UNARY_2D(func_name, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_BF16_2D(func_name)\n\
\n\
//EXP\n\
ADD_ELTSISE_UNARY_2D(exp)\n\
//SIN\n\
ADD_ELTSISE_UNARY_2D(sin)\n\
//COS\n\
ADD_ELTSISE_UNARY_2D(cos)\n\
//LOG\n\
ADD_ELTSISE_UNARY_2D(log)\n\
//SELU\n\
ADD_ELTSISE_UNARY_2D(selu)\n\
//NEG\n\
ADD_ELTSISE_UNARY_2D(neg)\n\
//CELU\n\
ADD_ELTSISE_UNARY_2D(celu)\n\
//RCP\n\
ADD_ELTSISE_UNARY_2D(rcp)\n\
//SIGN\n\
ADD_ELTSISE_UNARY_2D(sign)\n\
//SOFTSIGN\n\
ADD_ELTSISE_UNARY_2D(softsign)\n\
//ATAN\n\
ADD_ELTSISE_UNARY_2D(atan)\n\
//ATANH\n\
ADD_ELTSISE_UNARY_2D(atanh)\n\
//ACOSH\n\
ADD_ELTSISE_UNARY_2D(acosh)\n\
//TAN\n\
ADD_ELTSISE_UNARY_2D(tan)\n\
"; /* end of eltwise_unary_2d_1_vx*/

static const char eltwise_unary_3d_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float alpha;\n\
_viv_uniform float beta;\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
#define rlogE    (0.693147182f)\n\
\n\
float4 eltwise_unary_hard_sigmoid(float4 x)\n\
{\n\
    x = alpha * x + beta;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
float4 _softrelu(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    x += 1;\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 _tanh(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return (2 * x - 1);\n\
}\n\
\n\
float4 eltwise_unary_mish(float4 x)\n\
{\n\
    float4 y = _softrelu(x);\n\
    x = x * _tanh(y);\n\
    return x;\n\
}\n\
\n\
float4 eltwise_unary_round(float4 x)\n\
{\n\
    return convert_float4(convert_int4_rte(x));\n\
}\n\
\n\
float4 evaluate_polynomial_alpha(float4 x2)\n\
{\n\
    float4 alpha0 = (float4){-2.72614225801306e-10f, 2.77068142495902e-08f,\n\
                            -2.10102402082508e-06f, -5.69250639462346e-05f};\n\
    float4 alpha1 = (float4){-7.34990630326855e-04f, -2.95459980854025e-03f,\n\
                            -1.60960333262415e-02f, 0};\n\
\n\
    float4 poly = alpha0.x * x2 + alpha0.y;\n\
    poly = poly * x2 + alpha0.z;\n\
    poly = poly * x2 + alpha0.w;\n\
    poly = poly * x2 + alpha1.x;\n\
    poly = poly * x2 + alpha1.y;\n\
    poly = poly * x2 + alpha1.z;\n\
\n\
    return poly;\n\
}\n\
\n\
float4 evaluate_polynomial_beta(float4 x2)\n\
{\n\
    float4 beta0 = (float4){-1.45660718464996e-05f, -2.13374055278905e-04f,\n\
                            -1.68282697438203e-03f, -7.37332916720468e-03f};\n\
    float4 beta1 = (float4){-1.42647390514189e-02f, 0, 0, 0};\n\
\n\
    float4 poly = beta0.x * x2 + beta0.y;\n\
    poly = poly * x2 + beta0.z;\n\
    poly = poly * x2 + beta0.w;\n\
    poly = poly * x2 + beta1.x;\n\
\n\
    return 1.0f / poly;\n\
}\n\
\n\
float4 erf_eval(float4 _x)\n\
{\n\
    float4 x = clamp(_x, -4, 4);\n\
    float4 x2 = x * x;\n\
\n\
    return x * evaluate_polynomial_alpha(x2) * evaluate_polynomial_beta(x2);\n\
}\n\
\n\
#define RSQRT2      (0.70710678118654752440084436210485f)\n\
float4 eltwise_unary_gelu(float4 x)\n\
{\n\
    float4 erf, data;\n\
    data = x * RSQRT2;\n\
    erf = erf_eval(data);\n\
    x = 0.5f * x * (1 + erf);\n\
\n\
    return x;\n\
}\n\
\n\
#define SQRT_2_RCP_PI  0.7978845834732056f\n\
float4 eltwise_unary_hard_gelu(float4 x)\n\
{\n\
    float4 cdf = 0.5f + 0.5f * _tanh(SQRT_2_RCP_PI *\n\
                        (x + 0.044715f * x * x * x));\n\
    return x * cdf;\n\
}\n\
\n\
float4 eltwise_unary_inverse_sigmoid(float4 x)\n\
{\n\
    float4 x1, x2;\n\
    x = clamp(x, 0, 1);\n\
    x1 = clamp(x, alpha, 1);\n\
    x2 = clamp((1 - x), alpha, 1);\n\
    return log(x1 / x2);\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define ELTSISE_UNARY_3D(func_name, src_type_name, dst_type_name, src_type, \\\n\
                src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
__kernel void func_name##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecB = vecB * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
    vecB = vecB * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//MISH\n\
ELTSISE_UNARY_3D(mish, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(mish, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(mish, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(mish, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(mish, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(mish, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(mish, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(mish, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(mish, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(mish, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//ROUND\n\
ELTSISE_UNARY_3D(round, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(round, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(round, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(round, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(round, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(round, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(round, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(round, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(round, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(round, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//GELU\n\
ELTSISE_UNARY_3D(gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_GELU\n\
ELTSISE_UNARY_3D(hard_gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//INVERSE_SIGMOID\n\
ELTSISE_UNARY_3D(inverse_sigmoid, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(inverse_sigmoid, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
#define ELTSISE_UNARY_BF16(func_name) \\\n\
    __kernel void func_name##_BF16toBF16( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//MISH\n\
ELTSISE_UNARY_BF16(mish)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_BF16(hard_sigmoid)\n\
//ROUND\n\
ELTSISE_UNARY_BF16(round)\n\
//GELU\n\
ELTSISE_UNARY_BF16(gelu)\n\
//HARD_GELU\n\
ELTSISE_UNARY_BF16(hard_gelu)\n\
//INVERSE_SIGMOID\n\
ELTSISE_UNARY_BF16(inverse_sigmoid)"; /* end of eltwise_unary_3d_0_vx*/

static const char eltwise_unary_3d_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float alpha;\n\
_viv_uniform float beta;\n\
\n\
float4 eltwise_unary_sin(float4 x)\n\
{\n\
    return native_sin(x);\n\
}\n\
\n\
float4 eltwise_unary_cos(float4 x)\n\
{\n\
    return native_cos(x);\n\
}\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
float4 eltwise_unary_exp(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
#define rlogE    (0.693147182f)\n\
float4 eltwise_unary_log(float4 x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 eltwise_unary_neg(float4 x)\n\
{\n\
    return x * -1;\n\
}\n\
\n\
float4 eltwise_unary_selu(float4 val)\n\
{\n\
    float4 x = val * logE;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val * beta;\n\
}\n\
\n\
float4 eltwise_unary_celu(float4 val)\n\
{\n\
    float4 x = val * logE * beta;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val;\n\
}\n\
\n\
float4 eltwise_unary_rcp(float4 val)\n\
{\n\
    return 1.0f / val;\n\
}\n\
\n\
float4 eltwise_unary_sign(float4 val)\n\
{\n\
    return sign(val);\n\
}\n\
\n\
float4 eltwise_unary_softsign(float4 val)\n\
{\n\
    float4 _rcp = 1.0f / (1.0f + fabs(val));\n\
    return val * _rcp;\n\
}\n\
\n\
float4 eltwise_unary_atan(float4 val)\n\
{\n\
    return atan(val);\n\
}\n\
\n\
float4 eltwise_unary_atanh(float4 val)\n\
{\n\
    return atanh(val);\n\
}\n\
\n\
float4 eltwise_unary_acosh(float4 val)\n\
{\n\
    return acosh(val);\n\
}\n\
\n\
float4 eltwise_unary_tan(float4 val)\n\
{\n\
    return native_tan(val);\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define ELTSISE_UNARY_3D(func_name, src_type_name, dst_type_name, src_type, \\\n\
                src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
__kernel void func_name##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecB = vecB * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
    vecB = vecB * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
#define ELTSISE_UNARY_BF16(func_name) \\\n\
    __kernel void func_name##_BF16toBF16( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
#define ADD_ELTSISE_UNARY_3D(func_name) \\\n\
ELTSISE_UNARY_3D(func_name, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_3D(func_name, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8) \\\n\
ELTSISE_UNARY_3D(func_name, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8) \\\n\
ELTSISE_UNARY_3D(func_name, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8) \\\n\
ELTSISE_UNARY_3D(func_name, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8) \\\n\
ELTSISE_UNARY_3D(func_name, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_3D(func_name, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8) \\\n\
ELTSISE_UNARY_3D(func_name, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_3D(func_name, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8) \\\n\
ELTSISE_UNARY_3D(func_name, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8) \\\n\
ELTSISE_UNARY_BF16(func_name)\n\
\n\
//EXP\n\
ADD_ELTSISE_UNARY_3D(exp)\n\
//SIN\n\
ADD_ELTSISE_UNARY_3D(sin)\n\
//COS\n\
ADD_ELTSISE_UNARY_3D(cos)\n\
//LOG\n\
ADD_ELTSISE_UNARY_3D(log)\n\
//SELU\n\
ADD_ELTSISE_UNARY_3D(selu)\n\
//NEG\n\
ADD_ELTSISE_UNARY_3D(neg)\n\
//CELU\n\
ADD_ELTSISE_UNARY_3D(celu)\n\
//RCP\n\
ADD_ELTSISE_UNARY_3D(rcp)\n\
//SIGN\n\
ADD_ELTSISE_UNARY_3D(sign)\n\
//SOFTSIGN\n\
ADD_ELTSISE_UNARY_3D(softsign)\n\
//ATAN\n\
ADD_ELTSISE_UNARY_3D(atan)\n\
//ATANH\n\
ADD_ELTSISE_UNARY_3D(atanh)\n\
//ACOSH\n\
ADD_ELTSISE_UNARY_3D(acosh)\n\
//TAN\n\
ADD_ELTSISE_UNARY_3D(tan)\n\
"; /* end of eltwise_unary_3d_1_vx*/

static const char erf_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
float4 evaluate_polynomial_alpha(float4 x2)\n\
{\n\
    float4 alpha0 = (float4){-2.72614225801306e-10f, 2.77068142495902e-08f,\n\
                            -2.10102402082508e-06f, -5.69250639462346e-05f};\n\
    float4 alpha1 = (float4){-7.34990630326855e-04f, -2.95459980854025e-03f,\n\
                            -1.60960333262415e-02f, 0};\n\
\n\
    float4 poly = alpha0.x * x2 + alpha0.y;\n\
    poly = poly * x2 + alpha0.z;\n\
    poly = poly * x2 + alpha0.w;\n\
    poly = poly * x2 + alpha1.x;\n\
    poly = poly * x2 + alpha1.y;\n\
    poly = poly * x2 + alpha1.z;\n\
\n\
    return poly;\n\
}\n\
\n\
float4 evaluate_polynomial_beta(float4 x2)\n\
{\n\
    float4 beta0 = (float4){-1.45660718464996e-05f, -2.13374055278905e-04f,\n\
                            -1.68282697438203e-03f, -7.37332916720468e-03f};\n\
    float4 beta1 = (float4){-1.42647390514189e-02f, 0, 0, 0};\n\
\n\
    float4 poly = beta0.x * x2 + beta0.y;\n\
    poly = poly * x2 + beta0.z;\n\
    poly = poly * x2 + beta0.w;\n\
    poly = poly * x2 + beta1.x;\n\
\n\
    return 1.0f / poly;\n\
}\n\
\n\
float4 eltwise_unary_erf(float4 _x)\n\
{\n\
    float4 x = clamp(_x, -4, 4);\n\
    float4 x2 = x * x;\n\
\n\
    return x * evaluate_polynomial_alpha(x2) * evaluate_polynomial_beta(x2);\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
\n\
#define ELTSISE_UNARY_2D(func_name, src_type_name, dst_type_name, src_type, \\\n\
        src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
    __kernel void func_name##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//ERF\n\
ELTSISE_UNARY_2D(erf, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(erf, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(erf, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(erf, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(erf, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(erf, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(erf, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(erf, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(erf, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(erf, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define ELTSISE_UNARY_BF16_2D(func_name) \\\n\
    __kernel void func_name##_BF16toBF16_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//EXP\n\
ELTSISE_UNARY_BF16_2D(erf)\n\
\n\
#define ELTSISE_UNARY_3D(func_name, src_type_name, dst_type_name, src_type, \\\n\
    src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
__kernel void func_name##_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output \\\n\
) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//ERF\n\
ELTSISE_UNARY_3D(erf, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(erf, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(erf, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(erf, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(erf, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(erf, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(erf, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(erf, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(erf, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(erf, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
#define ELTSISE_UNARY_BF16_3D(func_name) \\\n\
    __kernel void func_name##_BF16toBF16( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//ERF\n\
ELTSISE_UNARY_BF16_3D(erf)"; /* end of erf_vx*/

static const char extra_ending_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void extra_ending_I16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void extra_ending_F16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void extra_ending_I8\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void extra_ending_U8\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of extra_ending_vx*/

static const char floordiv_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertFstToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecToFp32_4x4;\n\
\n\
_viv_uniform float in_scale0;\n\
_viv_uniform float in_scale1;\n\
_viv_uniform float out_scale;\n\
_viv_uniform float in0Tail;\n\
_viv_uniform float in1Tail;\n\
_viv_uniform float out_zp;\n\
\n\
#define FLOORDIV_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                      IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    save_type data; \\\n\
    read_type read_data0, read_data1; \\\n\
    copy_type tmpData0, tmpData1; \\\n\
    vxc_float4 in0Val1, in0Val2, in1Val1, in1Val2; \\\n\
    vxc_float4 tmpVal1, tmpVal2; \\\n\
    dst_type tmpOut1, tmpOut2; \\\n\
    read_fun(read_data0, input0, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmpData0, read_data0, 16); \\\n\
    read_fun(read_data1, input1, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmpData1, read_data1, 16); \\\n\
    VXC_DP4x4(in0Val1, tmpData0, tmpData0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
    VXC_DP4x4(in0Val2, tmpData0, tmpData0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
    VXC_DP4x4(in1Val1, tmpData1, tmpData1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
    VXC_DP4x4(in1Val2, tmpData1, tmpData1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
    in0Val1 = in0Val1 * IN0_SCALE + IN0_TAIL; \\\n\
    in0Val2 = in0Val2 * IN0_SCALE + IN0_TAIL; \\\n\
    in1Val1 = in1Val1 * IN1_SCALE + IN1_TAIL; \\\n\
    in1Val2 = in1Val2 * IN1_SCALE + IN1_TAIL; \\\n\
    tmpVal1 = floor(in0Val1 / in1Val1) * OUT_SCALE + OUT_OFFSET; \\\n\
    tmpVal2 = floor(in0Val2 / in1Val2) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, tmpOut1, tmpVal1); \\\n\
    _viv_asm(conv_mode, tmpOut2, tmpVal2); \\\n\
    VXC_DP2x8(data, tmpOut1, tmpOut2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
\n\
#define TENSOR_FLOORDIV(src0_name, src1_name, dst_name, dst_type, save_type, read_type, copy_type, \\\n\
                    conv_mode, IN0_SCALE, IN0_TAIL, IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET) \\\n\
__kernel void floordiv_##src0_name##src1_name##to##dst_name \\\n\
    ( \\\n\
    image2d_array_t input0, \\\n\
    image2d_array_t input1, \\\n\
    image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    FLOORDIV_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                  IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
\n\
TENSOR_FLOORDIV(F16, F16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_half8, CONV, 1, 0, 1, 0, 1, 0)\n\
TENSOR_FLOORDIV(F16, F16, I16, short4, vxc_short8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_FLOORDIV(F16, F16, I8,  char4, vxc_char8, vxc_short8,\\\n\
                vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_FLOORDIV(F16, F16, U8,  uchar4, vxc_uchar8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
\n\
TENSOR_FLOORDIV(I16, I16, I16, short4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV(I16, I16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_FLOORDIV(I8, I8, I8, char4, vxc_char8, vxc_char16,\\\n\
                vxc_char16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV(I8, I8, F16, half4, vxc_short8, vxc_char16,\\\n\
                vxc_char16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_FLOORDIV(U8, U8, U8,  uchar4, vxc_uchar8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV(U8, U8, F16, half4, vxc_short8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
\n\
#define TENSOR_FLOORDIV_2D(src0_name, src1_name, dst_name, dst_type, save_type, read_type, copy_type, \\\n\
    conv_mode, IN0_SCALE, IN0_TAIL, IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET) \\\n\
__kernel void floordiv_##src0_name##src1_name##to##dst_name##_2D \\\n\
    ( \\\n\
    image2d_array_t input0, \\\n\
    image2d_array_t input1, \\\n\
    image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    FLOORDIV_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                  IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, VXC_ReadImage, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
TENSOR_FLOORDIV_2D(F16, F16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_half8, CONV, 1, 0, 1, 0, 1, 0)\n\
TENSOR_FLOORDIV_2D(F16, F16, I16, short4, vxc_short8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_FLOORDIV_2D(F16, F16, I8,  char4, vxc_char8, vxc_short8,\\\n\
                vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_FLOORDIV_2D(F16, F16, U8,  uchar4, vxc_uchar8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
\n\
TENSOR_FLOORDIV_2D(I16, I16, I16, short4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV_2D(I16, I16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_FLOORDIV_2D(I8, I8, I8, char4, vxc_char8, vxc_char16,\\\n\
                vxc_char16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV_2D(I8, I8, F16, half4, vxc_short8, vxc_char16,\\\n\
                vxc_char16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_FLOORDIV_2D(U8, U8, U8,  uchar4, vxc_uchar8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV_2D(U8, U8, F16, half4, vxc_short8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define FLOORDIV_BF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 read_data0, read_data1, vec0; \\\n\
    vxc_float4 in0Val1, in0Val2, in1Val1, in1Val2; \\\n\
    vxc_float4 tmpVal1, tmpVal2; \\\n\
    vxc_ushort8 dst0, dst1; \\\n\
    vxc_ushort8 vect; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    read_fun(read_data0, input0, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(vec0, read_data0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, in0Val1, vec0, 16); \\\n\
    VXC_DP2x8(vec0, read_data0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, in0Val2, vec0, 16); \\\n\
    read_fun(read_data1, input1, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(vec0, read_data1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, in1Val1, vec0, 16); \\\n\
    VXC_DP2x8(vec0, read_data1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, in1Val2, vec0, 16); \\\n\
    tmpVal1 = floor(in0Val1 / in1Val1); \\\n\
    tmpVal2 = floor(in0Val2 / in1Val2); \\\n\
    _viv_asm(COPY, dst0, tmpVal1, 16); \\\n\
    _viv_asm(COPY, dst1, tmpVal2, 16); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void floordiv_BF16BF16toBF16\n\
    (\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    FLOORDIV_BF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray);\n\
}\n\
\n\
__kernel void floordiv_BF16BF16toBF16_2D\n\
    (\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    FLOORDIV_BF16_PROCESS(VXC_ReadImage, VXC_WriteImage);\n\
}"; /* end of floordiv_vx*/

static const char gather_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
_viv_uniform VXC_512Bits uniExtraCopyDpKeepinEvis_2x8;\n\
\n\
__kernel void gather_I8toI8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_U8toU8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_I16toI16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_I8toI8_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_char16 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_U8toU8_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_uchar16 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_I16toI16_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_F16toF16_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_vx*/

static const char gather_array_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
_viv_uniform int remainder;\n\
_viv_uniform int width;\n\
_viv_uniform VXC_512Bits uniExtraCopyDpKeepinEvis_2x8;\n\
\n\
__kernel void gather_I8toI8_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
\n\
    if (gidx == ((block_size >> 4) * 16))\n\
    {\n\
        __global char* data_ptr = (__global char*)input_ptr;\n\
        __global char* dst_ptr = (__global char*)output_ptr;\n\
        int i = 0;\n\
        for (i = 0; i < block_size - gidx; i ++)\n\
        {\n\
           dst_ptr[i] = data_ptr[i];\n\
        }\n\
    }\n\
    else\n\
    {\n\
        __global vxc_char16* data_ptr = (__global vxc_char16*)input_ptr;\n\
        vxc_char16 src = data_ptr[0];\n\
        __global vxc_char16* dst_ptr = (__global vxc_char16*)output_ptr;\n\
        dst_ptr[0] = src;\n\
    }\n\
}\n\
\n\
__kernel void gather_U8toU8_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
\n\
    if (gidx == ((block_size >> 4) * 16))\n\
    {\n\
        __global uchar* data_ptr = (__global uchar*)input_ptr;\n\
        __global uchar* dst_ptr = (__global uchar*)output_ptr;\n\
        int i = 0;\n\
        for (i = 0; i < block_size - gidx; i ++)\n\
        {\n\
           dst_ptr[i] = data_ptr[i];\n\
        }\n\
    }\n\
    else\n\
    {\n\
        __global vxc_uchar16* data_ptr = (__global vxc_uchar16*)input_ptr;\n\
        vxc_uchar16 src = data_ptr[0];\n\
        __global vxc_uchar16* dst_ptr = (__global vxc_uchar16*)output_ptr;\n\
        dst_ptr[0] = src;\n\
    }\n\
}\n\
\n\
__kernel void gather_I16toI16_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
\n\
    if (gidx == ((block_size >> 3) * 8))\n\
    {\n\
        __global short* data_ptr = (__global short*)input_ptr;\n\
        __global short* dst_ptr = (__global short*)output_ptr;\n\
        int i = 0;\n\
        for (i = 0; i < block_size - gidx; i ++)\n\
        {\n\
           dst_ptr[i] = data_ptr[i];\n\
        }\n\
    }\n\
    else\n\
    {\n\
        __global vxc_short8* data_ptr = (__global vxc_short8*)input_ptr;\n\
        vxc_short8 src = data_ptr[0];\n\
        __global vxc_short8* dst_ptr = (__global vxc_short8*)output_ptr;\n\
        dst_ptr[0] = src;\n\
    }\n\
}\n\
\n\
__kernel void gather_F16toF16_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
\n\
    if (gidx == ((block_size >> 3) * 8))\n\
    {\n\
        __global short* data_ptr = (__global short*)input_ptr;\n\
        __global short* dst_ptr = (__global short*)output_ptr;\n\
        int i = 0;\n\
        for (i = 0; i < block_size - gidx; i ++)\n\
        {\n\
           dst_ptr[i] = data_ptr[i];\n\
        }\n\
    }\n\
    else\n\
    {\n\
        __global vxc_short8* data_ptr = (__global vxc_short8*)input_ptr;\n\
        vxc_short8 src = data_ptr[0];\n\
        __global vxc_short8* dst_ptr = (__global vxc_short8*)output_ptr;\n\
        dst_ptr[0] = src;\n\
    }\n\
}\n\
\n\
#define GATHER_AXIS0_ARRAY(src0_type_name, read_type, data_type, write_type) \\\n\
__kernel void gather_##src0_type_name##to##src0_type_name##_axis0_array( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    if (coord.x >= width) return; \\\n\
    Image img0 = create_image_from_image2d(input0, 1); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    Image img2 = create_image_from_image2d(output, 1); \\\n\
    uchar* index_ptr = get_image_ptr_from_coord(img1, coord.xz); \\\n\
    __global int* index = (__global int*)index_ptr; \\\n\
    int4 indices = vload4(0, index); \\\n\
 \\\n\
    read_type src, dst; \\\n\
 \\\n\
    uchar* input_ptr = get_image_ptr_from_coord(img0, coord.zy); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.xy); \\\n\
    __global data_type* data_ptr = (__global data_type*)input_ptr; \\\n\
    __global write_type* out_ptr = (__global write_type*)output_ptr; \\\n\
    indices = indices >= 0 ? indices : indices + axis_num; \\\n\
    if (coord.x + remainder < width) \\\n\
    { \\\n\
        src.s0 = data_ptr[indices.x]; \\\n\
        src.s1 = data_ptr[indices.y]; \\\n\
        src.s2 = data_ptr[indices.z]; \\\n\
        src.s3 = data_ptr[indices.w]; \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        __global data_type* out_ptr_remainder = (__global data_type*)output_ptr; \\\n\
        switch (remainder) \\\n\
        { \\\n\
            case 3: \\\n\
                out_ptr_remainder[2] = data_ptr[indices.z]; \\\n\
            case 2: \\\n\
                out_ptr_remainder[1] = data_ptr[indices.y]; \\\n\
            case 1: \\\n\
                out_ptr_remainder[0] = data_ptr[indices.x]; \\\n\
                break; \\\n\
            default: \\\n\
                break; \\\n\
        } \\\n\
        return; \\\n\
    } \\\n\
 \\\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                     uniExtraCopyDpKeepinEvis_2x8); \\\n\
    out_ptr[0] = dst.s0123; \\\n\
}\n\
GATHER_AXIS0_ARRAY(U8, vxc_uchar16, uchar, vxc_uchar4)\n\
GATHER_AXIS0_ARRAY(I8, vxc_char16,  char, vxc_char4)\n\
GATHER_AXIS0_ARRAY(I16, vxc_short8, short, vxc_short4)\n\
GATHER_AXIS0_ARRAY(F16, vxc_short8, short, vxc_short4)"; /* end of gather_array_vx*/

static const char gather_batch_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
_viv_uniform VXC_512Bits uniExtraCopyDpKeepinEvis_2x8;\n\
_viv_uniform int batch;\n\
\n\
__kernel void gather_batch_I8toI8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        vxc_char16 src;\n\
        VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.z++;\n\
        VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
__kernel void gather_batch_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        vxc_uchar16 src;\n\
        VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.z++;\n\
        VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
__kernel void gather_batch_I16toI16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        vxc_short8 src;\n\
        VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.z++;\n\
        VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
__kernel void gather_batch_F16toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        vxc_short8 src;\n\
        VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.z++;\n\
        VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
__kernel void gather_batch_I8toI8_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 indices = read_imagei(input1, coord.xz);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int4 coord_in = (int4)(indices.x, get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char16 src, dst;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.y;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.z;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.w;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_batch_U8toU8_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 indices = read_imagei(input1, coord.xz);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int4 coord_in = (int4)(indices.x, get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar16 src, dst;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.y;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.z;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.w;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_batch_I16toI16_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 indices = read_imagei(input1, coord.xz);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int4 coord_in = (int4)(indices.x, get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.y;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.z;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.w;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_batch_F16toF16_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 indices = read_imagei(input1, coord.xz);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int4 coord_in = (int4)(indices.x, get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.y;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.z;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.w;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_batch_vx*/

static const char gather_elements_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int axis_size;\n\
_viv_uniform uint width0;\n\
_viv_uniform uint height0;\n\
_viv_uniform uint width1;\n\
_viv_uniform uint height1;\n\
_viv_uniform uint width_out;\n\
_viv_uniform uint height_out;\n\
\n\
#define GATHER_ELEMENTS_AXIS0_2D(name, data_type) \\\n\
__kernel void gather_elements_axis0_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
    int4 indice1 = indice + axis_size; \\\n\
    indice = indice < 0 ? indice1 : indice; \\\n\
 \\\n\
    data_type src; \\\n\
    VXC_ReadImage(src, input0, (int2)(indice.x, coord.y), 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, (int2)(indice.y, coord.y), 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, (int2)(indice.z, coord.y), 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, (int2)(indice.w, coord.y), 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ELEMENTS_AXIS0_2D(F16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS0_2D(I16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS0_2D(I8,  vxc_char4)\n\
GATHER_ELEMENTS_AXIS0_2D(U8,  vxc_uchar4)\n\
\n\
#define GATHER_ELEMENTS_AXIS0(name, data_type) \\\n\
__kernel void gather_elements_axis0_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4); \\\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
    int4 indice1 = indice + axis_size; \\\n\
    indice = indice < 0 ? indice1 : indice; \\\n\
 \\\n\
    data_type src; \\\n\
    int4 coord_in = coord; \\\n\
    coord_in.x = indice.x; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, \\\n\
                VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indice.y; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, \\\n\
                VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indice.z; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, \\\n\
                VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indice.w; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, \\\n\
                VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ELEMENTS_AXIS0(F16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS0(I16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS0(I8,  vxc_char4)\n\
GATHER_ELEMENTS_AXIS0(U8,  vxc_uchar4)\n\
\n\
#define GATHER_ELEMENTS_AXIS1_2D(name, data_type) \\\n\
__kernel void gather_elements_axis1_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type src; \\\n\
    VXC_ReadImage(src, input0, (int2)(coord.x, index), 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ELEMENTS_AXIS1_2D(F16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS1_2D(I16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS1_2D(I8,  vxc_char4)\n\
GATHER_ELEMENTS_AXIS1_2D(U8,  vxc_uchar4)\n\
\n\
#define GATHER_ELEMENTS_AXIS1(name, data_type) \\\n\
__kernel void gather_elements_axis1_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type src; \\\n\
    int4 coord_in = coord; \\\n\
    coord_in.y = index; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, \\\n\
                VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ELEMENTS_AXIS1(F16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS1(I16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS1(I8,  vxc_char4)\n\
GATHER_ELEMENTS_AXIS1(U8,  vxc_uchar4)\n\
\n\
#define GATHER_ELEMENTS_AXIS2(name, data_type) \\\n\
__kernel void gather_elements_axis2_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type src; \\\n\
    int4 coord_in = coord; \\\n\
    coord_in.z = index; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, \\\n\
                VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ELEMENTS_AXIS2(F16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS2(I16, vxc_short4)\n\
GATHER_ELEMENTS_AXIS2(I8,  vxc_char4)\n\
GATHER_ELEMENTS_AXIS2(U8,  vxc_uchar4)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis0_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int    axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    Tensor index_tensor = create_tensor_from_image2d_array(input1, 4); \\\n\
    int* index_ptr = (int*)index_tensor.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1 + coord.z * width1 * height1]; \\\n\
 \\\n\
    Tensor input_tensor = create_tensor_from_image2d_array(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_tensor.ptr; \\\n\
    data_type data = input_ptr[index + coord.y * width0 + coord.z * width0 * height0]; \\\n\
 \\\n\
    Tensor output_tensor = create_tensor_from_image2d_array(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_tensor.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out + coord.z * width_out * height_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(U8,  uchar, uchar*, 1)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis1_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int    axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    Tensor index_tensor = create_tensor_from_image2d_array(input1, 4); \\\n\
    int* index_ptr = (int*)index_tensor.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1 + coord.z * width1 * height1]; \\\n\
 \\\n\
    Tensor input_tensor = create_tensor_from_image2d_array(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_tensor.ptr; \\\n\
    data_type data = input_ptr[coord.x + index * width0 + coord.z * width0 * height0]; \\\n\
 \\\n\
    Tensor output_tensor = create_tensor_from_image2d_array(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_tensor.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out + coord.z * width_out * height_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(U8,  uchar, uchar*, 1)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis2_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int    axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    Tensor index_tensor = create_tensor_from_image2d_array(input1, 4); \\\n\
    int* index_ptr = (int*)index_tensor.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1 + coord.z * width1 * height1]; \\\n\
 \\\n\
    Tensor input_tensor = create_tensor_from_image2d_array(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_tensor.ptr; \\\n\
    data_type data = input_ptr[coord.x + coord.y * width0 + index * width0 * height0]; \\\n\
 \\\n\
    Tensor output_tensor = create_tensor_from_image2d_array(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_tensor.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out + coord.z * width_out * height_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(U8,  uchar, uchar*, 1)\n\
\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis0_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
    int    axis \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    Image index_img = create_image_from_image2d(input1, 4); \\\n\
    int* index_ptr = (int*)index_img.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1]; \\\n\
 \\\n\
    Image input_img = create_image_from_image2d(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_img.ptr; \\\n\
    data_type data = input_ptr[index + coord.y * width0]; \\\n\
 \\\n\
    Image output_img = create_image_from_image2d(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_img.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(U8,  uchar, uchar*, 1)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis1_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
    int    axis \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    Image index_img = create_image_from_image2d(input1, 4); \\\n\
    int* index_ptr = (int*)index_img.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1]; \\\n\
 \\\n\
    Image input_img = create_image_from_image2d(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_img.ptr; \\\n\
    data_type data = input_ptr[coord.x + index  * width0]; \\\n\
 \\\n\
    Image output_img = create_image_from_image2d(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_img.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(U8,  uchar, uchar*, 1)\n\
\n\
\n\
"; /* end of gather_elements_vx*/

static const char gather_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define GATHER_8BITS_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gather_##src0_type_name##toF16( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0); \\\n\
    int4 indice = read_imagei(input1, coord_in.xy); \\\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num; \\\n\
    coord_in.w = gidz * axis_num + indice.x; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy); \\\n\
    vxc_half8  src0, src1; \\\n\
    vxc_short8 dst0, dst1; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0, VXC_MODIFIER(0,7,0, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_DP2x8(src1,src,ms0, VXC_MODIFIER(0,7,8, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    _viv_asm(COPY, dst1, src1, 16); \\\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    VXC_WriteImage(output, coord, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_8BITS_TO_F16(U8, vxc_uchar16)\n\
GATHER_8BITS_TO_F16(I8, vxc_char16)\n\
\n\
#define GATHER_F16_TO_QINT(src1_type_name, write_type) \\\n\
__kernel void gather_F16to##src1_type_name( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0); \\\n\
 \\\n\
    int4 indice = read_imagei(input1, coord_in.xy); \\\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num; \\\n\
    coord_in.w = gidz * axis_num + indice.x; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy); \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_F16_TO_QINT(U8, vxc_uchar16)\n\
GATHER_F16_TO_QINT(I8, vxc_char16)\n\
GATHER_F16_TO_QINT(I16, vxc_short8)\n\
\n\
__kernel void gather_I16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
\n\
    vxc_half8  src0;\n\
    vxc_short8 dst0;\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
    _viv_asm(COPY, dst0, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define GATHER_8BITS_TO_F16_AXIS0(src0_type_name, read_type) \\\n\
__kernel void gather_##src0_type_name##toF16_axis0( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int4 indices = read_imagei(input1, coord.xx); \\\n\
    indices = indices >= 0 ? indices : indices + axis_num; \\\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1)); \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    indices.x = get_global_id(1); \\\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0, VXC_MODIFIER(0,7,0, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_8BITS_TO_F16_AXIS0(U8, vxc_uchar16)\n\
GATHER_8BITS_TO_F16_AXIS0(I8, vxc_char16)\n\
\n\
#define GATHER_F16_TO_QINT_AXIS0(src1_type_name, write_type) \\\n\
__kernel void gather_F16to##src1_type_name##_axis0( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int4 indices = read_imagei(input1, coord.xx); \\\n\
    indices = indices >= 0 ? indices : indices + axis_num; \\\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1)); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    indices.x = get_global_id(1); \\\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_F16_TO_QINT_AXIS0(U8, vxc_uchar16)\n\
GATHER_F16_TO_QINT_AXIS0(I8, vxc_char16)\n\
GATHER_F16_TO_QINT_AXIS0(I16, vxc_short8)\n\
\n\
__kernel void gather_I16toF16_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_half8  src0;\n\
    vxc_short8 dst0;\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
    _viv_asm(COPY, dst0, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_mix_vx*/

static const char gather_mix_batch_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
_viv_uniform int batch;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define GATHER_BATCH_8BITS_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gather_batch_##src0_type_name##toF16( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int2 coord_idx = (int2)(gidy, 0); \\\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0); \\\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0); \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
 \\\n\
    for(; coord_idx.y < batch;) \\\n\
    { \\\n\
        int4 indice = read_imagei(input1, coord_idx); \\\n\
        coord_idx.y++; \\\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num; \\\n\
        coord_in.y = gidz * axis_num + indice.x; \\\n\
 \\\n\
        read_type src; \\\n\
        VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.z++; \\\n\
        vxc_half8  src0, src1; \\\n\
        vxc_short8 dst0, dst1; \\\n\
 \\\n\
        VXC_DP2x8(src0,src,ms0, VXC_MODIFIER(0,7,0, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_DP2x8(src1,src,ms0, VXC_MODIFIER(0,7,8, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        _viv_asm(COPY, dst0, src0, 16); \\\n\
        _viv_asm(COPY, dst1, src1, 16); \\\n\
        VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
        VXC_WriteImage2DArray(output, coord, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
        coord.x = gidx; \\\n\
    } \\\n\
}\n\
GATHER_BATCH_8BITS_TO_F16(U8, vxc_uchar16)\n\
GATHER_BATCH_8BITS_TO_F16(I8, vxc_char16)\n\
\n\
#define GATHER_BATCH_F16_TO_QINT(src1_type_name, write_type) \\\n\
__kernel void gather_batch_F16to##src1_type_name( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int2 coord_idx = (int2)(gidy, 0); \\\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0); \\\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0); \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    for(; coord_idx.y < batch;) \\\n\
    { \\\n\
        int4 indice = read_imagei(input1, coord_idx); \\\n\
        coord_idx.y++; \\\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num; \\\n\
        coord_in.y = gidz * axis_num + indice.x; \\\n\
 \\\n\
        vxc_short8 src; \\\n\
        VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.z++; \\\n\
 \\\n\
        vxc_half8 data; \\\n\
        write_type dst; \\\n\
        _viv_asm(COPY, data, src, 16); \\\n\
        VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
}\n\
GATHER_BATCH_F16_TO_QINT(U8, vxc_uchar16)\n\
GATHER_BATCH_F16_TO_QINT(I8, vxc_char16)\n\
GATHER_BATCH_F16_TO_QINT(I16, vxc_short8)\n\
\n\
__kernel void gather_batch_I16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    vxc_half8  src0;\n\
    vxc_short8 dst0;\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        vxc_short8 src;\n\
        VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.z++;\n\
        VXC_DP2x8(src0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
        _viv_asm(COPY, dst0, src0, 16);\n\
        VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
#define GATHER_BATCH_8BITS_TO_F16_AXIS0(src0_type_name, read_type) \\\n\
__kernel void gather_batch_##src0_type_name##toF16_axis0( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 indices = read_imagei(input1, coord.xz); \\\n\
    indices = indices >= 0 ? indices : indices + axis_num; \\\n\
    int4 coord_in = (int4)(indices.x, get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indices.y; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indices.z; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indices.w; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0, VXC_MODIFIER(0,7,0, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_BATCH_8BITS_TO_F16_AXIS0(U8, vxc_uchar16)\n\
GATHER_BATCH_8BITS_TO_F16_AXIS0(I8, vxc_char16)\n\
\n\
#define GATHER_BATCH_F16_TO_QINT_AXIS0(src1_type_name, write_type) \\\n\
__kernel void gather_batch_F16to##src1_type_name##_axis0( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num, \\\n\
    int is_array \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 indices = read_imagei(input1, coord.xz); \\\n\
    indices = indices >= 0 ? indices : indices + axis_num; \\\n\
    int4 coord_in = (int4)(indices.x, get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indices.y; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indices.z; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = indices.w; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_BATCH_F16_TO_QINT_AXIS0(U8, vxc_uchar16)\n\
GATHER_BATCH_F16_TO_QINT_AXIS0(I8, vxc_char16)\n\
GATHER_BATCH_F16_TO_QINT_AXIS0(I16, vxc_short8)\n\
\n\
__kernel void gather_batch_I16toF16_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int is_array\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 indices = read_imagei(input1, coord.xz);\n\
    indices = indices >= 0 ? indices : indices + axis_num;\n\
    int4 coord_in = (int4)(indices.x, get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src, dst;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.y;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.z;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = indices.w;\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_half8  src0;\n\
    vxc_short8 dst0;\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
    _viv_asm(COPY, dst0, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_mix_batch_vx*/

static const char gather_nd_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_I8toI8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_I16toI16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_array_I8toI8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global char* data_ptr = (__global char*)input_ptr;\n\
    __global char* dst_ptr = (__global char*)output_ptr;\n\
    char src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global uchar* data_ptr = (__global uchar*)input_ptr;\n\
    __global uchar* dst_ptr = (__global uchar*)output_ptr;\n\
    uchar src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_I16toI16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
\n\
}\n\
\n\
__kernel void gather_nd_array_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord.zw);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
"; /* end of gather_nd_vx*/

static const char gather_nd_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_I8toI8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_U8toU8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_I16toI16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_array_I8toI8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, indice.xy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global char* data_ptr = (__global char*)input_ptr;\n\
    __global char* dst_ptr = (__global char*)output_ptr;\n\
    char src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_U8toU8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, indice.xy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global uchar* data_ptr = (__global uchar*)input_ptr;\n\
    __global uchar* dst_ptr = (__global uchar*)output_ptr;\n\
    uchar src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
\n\
}\n\
\n\
__kernel void gather_nd_array_I16toI16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, indice.xy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_F16toF16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, indice.xy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
"; /* end of gather_nd_2d_vx*/

static const char gather_nd_2d_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16toFp16_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt16_2x8;\n\
\n\
#define GATHER_ND_QINT_TO_F16_2D(src0_type_name, read_type) \\\n\
__kernel void gather_nd_##src0_type_name##toF16_2D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord.zy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_QINT_TO_F16_2D(U8, vxc_uchar16)\n\
GATHER_ND_QINT_TO_F16_2D(I8, vxc_char16)\n\
GATHER_ND_QINT_TO_F16_2D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_F16_TO_QINT_2D(src1_type_name, write_type) \\\n\
__kernel void gather_nd_F16to##src1_type_name##_2D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven,1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord.zy, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_F16_TO_QINT_2D(U8, vxc_uchar16)\n\
GATHER_ND_F16_TO_QINT_2D(I8, vxc_char16)\n\
GATHER_ND_F16_TO_QINT_2D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_ARRAY_QINT_TO_F16_2D(src0_type_name, read_type, ptr_type, stride) \\\n\
__kernel void gather_nd_array_##src0_type_name##toF16_2D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input0, stride); \\\n\
    Image img2 = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, indice.xy); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy); \\\n\
 \\\n\
    __global ptr_type data_ptr = (__global ptr_type)input_ptr; \\\n\
    __global vxc_short8* dst_ptr = (__global vxc_short8* )output_ptr; \\\n\
    read_type src = data_ptr[0]; \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    dst_ptr[0] = dst0; \\\n\
}\n\
GATHER_ND_ARRAY_QINT_TO_F16_2D(U8, vxc_uchar16, vxc_uchar16*, 1)\n\
GATHER_ND_ARRAY_QINT_TO_F16_2D(I8, vxc_char16, vxc_char16*, 1)\n\
GATHER_ND_ARRAY_QINT_TO_F16_2D(I16, vxc_short8, vxc_short8*, 2)\n\
\n\
#define GATHER_ND_ARRAY_F16_TO_QINT_2D(src1_type_name, write_type, ptr_type, stride) \\\n\
__kernel void gather_nd_array_F16to##src1_type_name##_2D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input0, 2); \\\n\
    Image img2 = create_image_from_image2d(output, stride); \\\n\
 \\\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, indice.xy); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy); \\\n\
 \\\n\
    __global vxc_short8* data_ptr = (__global vxc_short8*)input_ptr; \\\n\
    __global ptr_type dst_ptr = (__global ptr_type )output_ptr; \\\n\
    vxc_short8 src = data_ptr[0]; \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven,1),uniConvertFp16toU8_2x8); \\\n\
    dst_ptr[0] = dst; \\\n\
}\n\
GATHER_ND_ARRAY_F16_TO_QINT_2D(U8, vxc_uchar16, vxc_uchar16*, 1)\n\
GATHER_ND_ARRAY_F16_TO_QINT_2D(I8, vxc_char16, vxc_char16*, 1)\n\
GATHER_ND_ARRAY_F16_TO_QINT_2D(I16, vxc_short8, vxc_short8*, 2)"; /* end of gather_nd_2d_mix_vx*/

static const char gather_nd_3d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_I8toI8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_U8toU8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_I16toI16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_array_I8toI8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global char* data_ptr = (__global char*)input_ptr;\n\
    __global char* dst_ptr = (__global char*)output_ptr;\n\
    char src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_U8toU8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global uchar* data_ptr = (__global uchar*)input_ptr;\n\
    __global uchar* dst_ptr = (__global uchar*)output_ptr;\n\
    uchar src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
\n\
}\n\
\n\
__kernel void gather_nd_array_I16toI16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_F16toF16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
"; /* end of gather_nd_3d_vx*/

static const char gather_nd_3d_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define GATHER_ND_QINT_TO_F16_3D(src0_type_name, read_type) \\\n\
__kernel void gather_nd_##src0_type_name##toF16_3D( \\\n\
    __read_only image2d_array_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
    indice.w = 0; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord.zy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_QINT_TO_F16_3D(U8, vxc_uchar16)\n\
GATHER_ND_QINT_TO_F16_3D(I8, vxc_char16)\n\
GATHER_ND_QINT_TO_F16_3D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_F16_TO_QINT_3D(src1_type_name, write_type) \\\n\
__kernel void gather_nd_F16to##src1_type_name##_3D( \\\n\
    __read_only image2d_array_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
    indice.w = 0; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven,1), uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord.zy, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_F16_TO_QINT_3D(U8, vxc_uchar16)\n\
GATHER_ND_F16_TO_QINT_3D(I8, vxc_char16)\n\
GATHER_ND_F16_TO_QINT_3D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_ARRAY_QINT_TO_F16_3D(src0_type_name, read_type, ptr_type, stride) \\\n\
__kernel void gather_nd_array_##src0_type_name##toF16_3D( \\\n\
    __read_only image2d_array_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
    indice.w = 0; \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, stride); \\\n\
    Image img2 = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy); \\\n\
 \\\n\
    __global ptr_type data_ptr = (__global ptr_type)input_ptr; \\\n\
    __global vxc_short8* dst_ptr = (__global vxc_short8* )output_ptr; \\\n\
    read_type src = data_ptr[0]; \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    dst_ptr[0] = dst0; \\\n\
}\n\
GATHER_ND_ARRAY_QINT_TO_F16_3D(U8, vxc_uchar16, vxc_uchar16*, 1)\n\
GATHER_ND_ARRAY_QINT_TO_F16_3D(I8, vxc_char16, vxc_char16*, 1)\n\
GATHER_ND_ARRAY_QINT_TO_F16_3D(I16, vxc_short8, vxc_short8*, 2)\n\
\n\
#define GATHER_ND_ARRAY_F16_TO_QINT_3D(src1_type_name, write_type, ptr_type, stride) \\\n\
__kernel void gather_nd_array_F16to##src1_type_name##_3D( \\\n\
    __read_only image2d_array_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
    indice.w = 0; \\\n\
 \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 2); \\\n\
    Image img2 = create_image_from_image2d(output, stride); \\\n\
 \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy); \\\n\
 \\\n\
    __global vxc_short8* data_ptr = (__global vxc_short8*)input_ptr; \\\n\
    __global ptr_type dst_ptr = (__global ptr_type )output_ptr; \\\n\
    vxc_short8 src = data_ptr[0]; \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven,1), uniConvertFp16toU8_2x8); \\\n\
    dst_ptr[0] = dst; \\\n\
}\n\
GATHER_ND_ARRAY_F16_TO_QINT_3D(U8, vxc_uchar16, vxc_uchar16*, 1)\n\
GATHER_ND_ARRAY_F16_TO_QINT_3D(I8, vxc_char16, vxc_char16*, 1)\n\
GATHER_ND_ARRAY_F16_TO_QINT_3D(I16, vxc_short8, vxc_short8*, 2)\n\
\n\
"; /* end of gather_nd_3d_mix_vx*/

static const char gather_nd_batch_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_batch_I8toI8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage(src, input0, coord0, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_batch_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage(src, input0, coord0, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_batch_I16toI16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord0, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_batch_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord0, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_array_batch_I8toI8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord0);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global char* data_ptr = (__global char*)input_ptr;\n\
    __global char* dst_ptr = (__global char*)output_ptr;\n\
    char src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_batch_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord0);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global uchar* data_ptr = (__global uchar*)input_ptr;\n\
    __global uchar* dst_ptr = (__global uchar*)output_ptr;\n\
    uchar src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_batch_I16toI16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord0);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_batch_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord0);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}"; /* end of gather_nd_batch_vx*/

static const char gather_nd_batch_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_batch_I8toI8_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_batch_U8toU8_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_batch_I16toI16_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_batch_F16toF16_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_array_batch_I8toI8_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 1);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 1);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global char* data_ptr = (__global char*)input_ptr;\n\
    __global char* dst_ptr = (__global char*)output_ptr;\n\
    char src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_batch_U8toU8_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 1);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 1);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global uchar* data_ptr = (__global uchar*)input_ptr;\n\
    __global uchar* dst_ptr = (__global uchar*)output_ptr;\n\
    uchar src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_batch_I16toI16_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_nd_array_batch_F16toF16_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index num\n\
    int gidz = get_global_id(2);  // batch num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    Tensor img = create_tensor_from_image2d_array(input1, 4);\n\
    uchar* indice_ptr = get_tensor_ptr_from_coord(img, coord.wyzw);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.zw = coord.zw;\n\
\n\
    Tensor img1 = create_tensor_from_image2d_array(input0, 2);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 2);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, indice);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global short* data_ptr = (__global short*)input_ptr;\n\
    __global short* dst_ptr = (__global short*)output_ptr;\n\
    short src = data_ptr[0];\n\
    dst_ptr[0] = src;\n\
}\n\
"; /* end of gather_nd_batch_2d_vx*/

static const char gather_nd_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16toFp16_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt16_2x8;\n\
\n\
#define GATHER_ND_QINT_TO_F16_1D(src0_type_name, read_type) \\\n\
__kernel void gather_nd_##src0_type_name##toF16_1D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    coord.w = indice.x; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord.zy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_QINT_TO_F16_1D(U8, vxc_uchar16)\n\
GATHER_ND_QINT_TO_F16_1D(I8, vxc_char16)\n\
GATHER_ND_QINT_TO_F16_1D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_F16_TO_QINT_1D(src1_type_name, write_type) \\\n\
__kernel void gather_nd_F16to##src1_type_name##_1D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    coord.w = indice.x; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord.zy, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_F16_TO_QINT_1D(U8, vxc_uchar16)\n\
GATHER_ND_F16_TO_QINT_1D(I8, vxc_char16)\n\
GATHER_ND_F16_TO_QINT_1D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_ARRAY_QINT_TO_F16_1D(src0_type_name, read_type, ptr_type, stride) \\\n\
__kernel void gather_nd_array_##src0_type_name##toF16_1D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    coord.w = indice.x; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input0, stride); \\\n\
    Image img2 = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord.zw); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy); \\\n\
 \\\n\
    __global ptr_type data_ptr = (__global ptr_type)input_ptr; \\\n\
    __global vxc_short8* dst_ptr = (__global vxc_short8* )output_ptr; \\\n\
    read_type src = data_ptr[0]; \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    dst_ptr[0] = dst0; \\\n\
}\n\
GATHER_ND_ARRAY_QINT_TO_F16_1D(U8, vxc_uchar16, vxc_uchar16*, 1)\n\
GATHER_ND_ARRAY_QINT_TO_F16_1D(I8, vxc_char16, vxc_char16*, 1)\n\
GATHER_ND_ARRAY_QINT_TO_F16_1D(I16, vxc_short8, vxc_short8*, 2)\n\
\n\
#define GATHER_ND_ARRAY_F16_TO_QINT_1D(src1_type_name, write_type, ptr_type, stride) \\\n\
__kernel void gather_nd_array_F16to##src1_type_name##_1D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    coord.w = indice.x; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input0, 2); \\\n\
    Image img2 = create_image_from_image2d(output, stride); \\\n\
 \\\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord.zw); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.zy); \\\n\
 \\\n\
    __global vxc_short8* data_ptr = (__global vxc_short8*)input_ptr; \\\n\
    __global ptr_type dst_ptr = (__global ptr_type )output_ptr; \\\n\
    vxc_short8 src = data_ptr[0]; \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    dst_ptr[0] = dst; \\\n\
}\n\
GATHER_ND_ARRAY_F16_TO_QINT_1D(U8, vxc_uchar16, vxc_uchar16*, 1)\n\
GATHER_ND_ARRAY_F16_TO_QINT_1D(I8, vxc_char16, vxc_char16*, 1)\n\
GATHER_ND_ARRAY_F16_TO_QINT_1D(I16, vxc_short8, vxc_short8*, 2)\n\
\n\
\n\
"; /* end of gather_nd_mix_vx*/

static const char get_matrix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float4 theta_1;\n\
_viv_uniform float4 theta_2;\n\
_viv_uniform float4 scale;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
\n\
#define GET_MATRIX_SH_IMPL(name0, in_type, read_func) \\\n\
__kernel void get_matrix_##name0##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 int       has_theta_1_1,  \\\n\
                 int       has_theta_1_2,  \\\n\
                 int       has_theta_1_3,  \\\n\
                 int       has_theta_2_1,  \\\n\
                 int       has_theta_2_2,  \\\n\
                 int       has_theta_2_3,  \\\n\
                 float     theta_1_1,  \\\n\
                 float     theta_1_2,  \\\n\
                 float     theta_1_3,  \\\n\
                 float     theta_2_1,  \\\n\
                 float     theta_2_2,  \\\n\
                 float     theta_2_3,  \\\n\
                 float     i_width, \\\n\
                 float     i_height, \\\n\
                 float     o_width, \\\n\
                 float     o_height \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(0, get_global_id(1)); \\\n\
    float4 matrix0, matrix1; \\\n\
    float4 theta1, theta2; \\\n\
    _viv_asm(COPY, theta1, theta_1, 16); \\\n\
    _viv_asm(COPY, theta2, theta_2, 16); \\\n\
 \\\n\
    if (has_theta_1_1 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta1.x = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_1_2 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta1.y = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_1_3 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta1.z = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_2_1 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta2.x = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_2_2 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta2.y = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_2_3 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta2.z = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    matrix0.x = theta2.y * scale.x; \\\n\
    matrix0.z = theta2.x * scale.z; \\\n\
    matrix1.x = ( theta2.z - theta2.y - theta2.x + 1) * i_width * 0.5f; \\\n\
    matrix0.y = theta1.y * scale.w; \\\n\
    matrix0.w = theta1.x * scale.y; \\\n\
    matrix1.y = ( theta1.z - theta1.y - theta1.x + 1) * i_height * 0.5f; \\\n\
    matrix1.zw = 2.0f * matrix0.xy; \\\n\
 \\\n\
    coord.x = 0; \\\n\
    vxc_ushort8 dst; \\\n\
    _viv_asm(COPY, dst, matrix0, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, dst, matrix1, 16); \\\n\
    coord.x = 8; \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GET_MATRIX_SH_IMPL(I16, int4,  read_imagei)\n\
GET_MATRIX_SH_IMPL(I8,  int4,  read_imagei)\n\
GET_MATRIX_SH_IMPL(U8,  uint4, read_imageui)\n\
\n\
__kernel void get_matrix_F16toF32\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 int       has_theta_1_1,\n\
                 int       has_theta_1_2,\n\
                 int       has_theta_1_3,\n\
                 int       has_theta_2_1,\n\
                 int       has_theta_2_2,\n\
                 int       has_theta_2_3,\n\
                 float     theta_1_1,\n\
                 float     theta_1_2,\n\
                 float     theta_1_3,\n\
                 float     theta_2_1,\n\
                 float     theta_2_2,\n\
                 float     theta_2_3,\n\
                 float     i_width,\n\
                 float     i_height,\n\
                 float     o_width,\n\
                 float     o_height\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(1));\n\
    float4 matrix0, matrix1;\n\
    float4 theta1, theta2;\n\
    _viv_asm(COPY, theta1, theta_1, 16);\n\
    _viv_asm(COPY, theta2, theta_2, 16);\n\
\n\
    if (has_theta_1_1 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta1.x = data.x;\n\
    }\n\
\n\
    if (has_theta_1_2 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta1.y = data.x;\n\
    }\n\
\n\
    if (has_theta_1_3 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta1.z = data.x;\n\
    }\n\
\n\
    if (has_theta_2_1 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta2.x = data.x;\n\
    }\n\
\n\
    if (has_theta_2_2 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta2.y = data.x;\n\
    }\n\
\n\
    if (has_theta_2_3 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta2.z = data.x;\n\
    }\n\
\n\
    matrix0.x = theta2.y * scale.x;\n\
    matrix0.z = theta2.x * scale.z;\n\
    matrix1.x = ( theta2.z - theta2.y - theta2.x + 1) * i_width * 0.5f;\n\
    matrix0.y = theta1.y * scale.w;\n\
    matrix0.w = theta1.x * scale.y;\n\
    matrix1.y = ( theta1.z - theta1.y - theta1.x + 1) * i_height * 0.5f;\n\
    matrix1.zw = 2.0f * matrix0.xy;\n\
\n\
    coord.x = 0;\n\
    vxc_ushort8 dst;\n\
    _viv_asm(COPY, dst, matrix0, 16);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, matrix1, 16);\n\
    coord.x = 8;\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of get_matrix_vx*/

static const char group_normalization_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_scale2;\n\
_viv_uniform float input_zp;\n\
_viv_uniform float sum_x_tail;\n\
_viv_uniform float sum_x2_tail0;\n\
_viv_uniform float sum_x2_tail1;\n\
\n\
_viv_uniform VXC_512Bits uniSumX_16x1;\n\
_viv_uniform VXC_512Bits uniSumX2_16x1;\n\
_viv_uniform VXC_512Bits uniResetFp32_4x4;\n\
_viv_uniform int group_stride;\n\
\n\
#define GROUP_NORM_SUMS_8BITS_IMPL(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sums_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int is2D) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    src_type src0; \\\n\
    float2 sums_f32 = 0; \\\n\
    int2 sums = 0, sum_x_x2; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
    if (gidx < width) \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
            VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
            sums = sums + sum_x_x2; \\\n\
        } \\\n\
        sums_f32 = convert_float2(sums); \\\n\
        sums_f32.y = sums_f32.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * sums_f32.x; \\\n\
        sums_f32.x = sums_f32.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
    lcl_sum[lidx] = sums_f32.x; \\\n\
    lcl_sqr[lidx] = sums_f32.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        float sum_x = 0,sum_x2 = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum_x += dot(tmp_sum[i], one); \\\n\
            sum_x2 += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 data = (float4)(sum_x, sum_x2, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
GROUP_NORM_SUMS_8BITS_IMPL(U8, vxc_uchar16)\n\
GROUP_NORM_SUMS_8BITS_IMPL(I8, vxc_char16)\n\
\n\
#define GROUP_NORM_SUMS_8BITS_IMPL_2D(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sums_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int is2D) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
 \\\n\
    int2 coord = (int2)(gidx, get_global_id(1)); \\\n\
    src_type src0; \\\n\
    float2 sums = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    if(gidx < width) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(sums, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sums, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        sums.y = sums.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * sums.x; \\\n\
        sums.x = sums.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
    lcl_sum[lidx] = sums.x; \\\n\
    lcl_sqr[lidx] = sums.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, get_global_id(1), 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        float sum_x = 0,sum_x2 = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum_x += dot(tmp_sum[i], one); \\\n\
            sum_x2 += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 data = (float4)(sum_x, sum_x2, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
GROUP_NORM_SUMS_8BITS_IMPL_2D(U8, vxc_uchar16)\n\
GROUP_NORM_SUMS_8BITS_IMPL_2D(I8, vxc_char16)\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_means(\n\
    image2d_t input, image2d_t output, float eps, float group_ratio)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_uchar16 src0 = 1;\n\
    float2 sum_sqr = (float2)(0);\n\
    float4 mean_vari;\n\
    VXC_DP4x4(mean_vari, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniResetFp32_4x4);\n\
\n\
    __local float2 lcl_data[16];\n\
    __local float2 lcl_sum[4];\n\
\n\
    for(; coord.x < group_stride; coord.x += 64)\n\
    {\n\
        mean_vari += read_imagef(input, coord);\n\
    }\n\
    lcl_data[lidx] = mean_vari.xy;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx < 4)\n\
    {\n\
        float2 tmpSum = (float2)(0);\n\
        for(int i = lidx; i < 16; i+=4)\n\
        {\n\
            tmpSum += lcl_data[i];\n\
        }\n\
        lcl_sum[lidx] = tmpSum;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx == 0)\n\
    {\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum_sqr += lcl_sum[i];\n\
        }\n\
        mean_vari.xy = sum_sqr * group_ratio;\n\
        mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
        mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
        coord.x = 0;\n\
        write_imagef(output, coord, mean_vari);\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_2_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_3_4x4;\n\
#define CONVERT_INPUT_TO_F32() \\\n\
VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4);\n\
\n\
#define GROUP_NORM_8BITS_IMPL(name, src_type, dst_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_array_t scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_para = (int4)((convert_int(gidx * rSpaceOrg) + convert_int(gidy * pStride)), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    dst_type dst; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    _viv_asm(COPY, scale_h, src1, 16); \\\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    float alpha = input_scale * output_scale * scale_vari; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    CONVERT_INPUT_TO_F32() \\\n\
    norm = tmpData0 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    norm = tmpData2 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8BITS_IMPL(U8_F16toU8, vxc_uchar16, vxc_uchar16)\n\
GROUP_NORM_8BITS_IMPL(I8_F16toI8, vxc_char16,  vxc_char16)\n\
\n\
#define GROUP_NORM_8BITS_IMPL_2D(name, src_type, dst_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_array_t scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int2 coord = (int2)(get_global_id(0), gidz); \\\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    dst_type dst; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    _viv_asm(COPY, scale_h, src1, 16); \\\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    float alpha = input_scale * output_scale * scale_vari; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    CONVERT_INPUT_TO_F32() \\\n\
    norm = tmpData0 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    norm = tmpData2 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8BITS_IMPL_2D(U8_F16toU8, vxc_uchar16, vxc_uchar16)\n\
GROUP_NORM_8BITS_IMPL_2D(I8_F16toI8, vxc_char16,  vxc_char16)\n\
\n\
#define GROUP_NORM_8BITS_F32_IMPL(name, src_type, dst_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_para = (int4)((convert_int(gidx * rSpaceOrg) + convert_int(gidy * pStride)), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    dst_type dst; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    scale_f = read_imagef(scale, coord_para.xy); \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    float alpha = input_scale * output_scale * scale_vari; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    CONVERT_INPUT_TO_F32() \\\n\
    norm = tmpData0 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    norm = tmpData2 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8BITS_F32_IMPL(U8_F32toU8, vxc_uchar16, vxc_uchar16)\n\
GROUP_NORM_8BITS_F32_IMPL(I8_F32toI8, vxc_char16,  vxc_char16)\n\
\n\
#define GROUP_NORM_8BITS_F32_IMPL_2D(name, src_type, dst_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int2 coord = (int2)(get_global_id(0), gidz); \\\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    dst_type dst; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    scale_f = read_imagef(scale, coord_para.xy); \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    float alpha = input_scale * output_scale * scale_vari; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    CONVERT_INPUT_TO_F32() \\\n\
    norm = tmpData0 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    norm = tmpData2 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8BITS_F32_IMPL_2D(U8_F32toU8, vxc_uchar16, vxc_uchar16)\n\
GROUP_NORM_8BITS_F32_IMPL_2D(I8_F32toI8, vxc_char16,  vxc_char16)\n\
"; /* end of group_normalization_0_vx*/

static const char group_normalization_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_2_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_3_4x4;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_zp;\n\
\n\
#define GROUP_NORM_8BITSTOF16_IMPL(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_array_t scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_para = (int4)((convert_int(gidx * rSpaceOrg) + convert_int(gidy * pStride)), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    vxc_short8 src1, outval; \\\n\
    vxc_half8 scale_h, dst; \\\n\
    float scale_vari, bias_val; \\\n\
    vxc_float4 bias_f, scale_f; \\\n\
 \\\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    _viv_asm(COPY, scale_h, src1, 16); \\\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float alpha = scale_vari * input_scale; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0); \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    norm = alpha * tmpData2 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData3 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8BITSTOF16_IMPL(U8_F16toF16, vxc_uchar16)\n\
GROUP_NORM_8BITSTOF16_IMPL(I8_F16toF16, vxc_char16)\n\
\n\
\n\
#define GROUP_NORM_8BITSTOF16_IMPL_2D(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_array_t scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int2 coord = (int2)(get_global_id(0), gidz); \\\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    vxc_short8 src1, outval; \\\n\
    vxc_half8 scale_h, dst; \\\n\
    float scale_vari, bias_val; \\\n\
    vxc_float4 bias_f, scale_f; \\\n\
 \\\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    _viv_asm(COPY, scale_h, src1, 16); \\\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float alpha = scale_vari; \\\n\
    alpha = scale_vari * input_scale; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0); \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    norm = alpha * tmpData2 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData3 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8BITSTOF16_IMPL_2D(U8_F16toF16, vxc_uchar16)\n\
GROUP_NORM_8BITSTOF16_IMPL_2D(I8_F16toF16, vxc_char16)\n\
\n\
#define GROUP_NORM_8TOF16_F32_IMPL(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_para = (int4)((convert_int(gidx * rSpaceOrg) + convert_int(gidy * pStride)), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    vxc_short8 outval; \\\n\
    vxc_half8 dst; \\\n\
    float scale_vari, bias_val; \\\n\
    vxc_float4 bias_f, scale_f; \\\n\
 \\\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    scale_f = read_imagef(scale, coord_para.xy); \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float alpha = scale_vari * input_scale; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0); \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    norm = alpha * tmpData2 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData3 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8TOF16_F32_IMPL(U8_F32toF16, vxc_uchar16)\n\
GROUP_NORM_8TOF16_F32_IMPL(I8_F32toF16, vxc_char16)\n\
\n\
#define GROUP_NORM_8TOF16_F32_IMPL_2D(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int2 coord = (int2)(get_global_id(0), gidz); \\\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    vxc_short8 outval; \\\n\
    vxc_half8 dst; \\\n\
    float scale_vari, bias_val; \\\n\
    vxc_float4 bias_f, scale_f; \\\n\
 \\\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    scale_f = read_imagef(scale, coord_para.xy); \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float alpha = scale_vari * input_scale; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0); \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    norm = alpha * tmpData2 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = alpha * tmpData3 + bias_val; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_8TOF16_F32_IMPL_2D(U8_F32toF16, vxc_uchar16)\n\
GROUP_NORM_8TOF16_F32_IMPL_2D(I8_F32toF16, vxc_char16)\n\
"; /* end of group_normalization_1_vx*/

static const char group_normalization_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniSum_X_X2_8x2;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_scale2;\n\
_viv_uniform float input_zp;\n\
_viv_uniform float sum_x_tail;\n\
_viv_uniform float sum_x2_tail0;\n\
_viv_uniform float sum_x2_tail1;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define GROUP_NORM_SUMS_16BITS_IMPL(name, load_type, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sums_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int is2D) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 3; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    load_type src; \\\n\
    src_type in_h; \\\n\
    float4 sumsqr; \\\n\
    float4 tmpSumSqr = (float4)(0); \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    if(gidx < width) \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            _viv_asm(COPY, in_h, src, 16); \\\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniSum_X_X2_8x2); \\\n\
            tmpSumSqr += sumsqr; \\\n\
        } \\\n\
        tmpSumSqr.y = tmpSumSqr.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * tmpSumSqr.x; \\\n\
        tmpSumSqr.x = tmpSumSqr.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
 \\\n\
    lcl_sum[lidx] = tmpSumSqr.x; \\\n\
    lcl_sqr[lidx] = tmpSumSqr.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        float sum = 0; \\\n\
        float sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 data = (float4)(sum, sqr, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
GROUP_NORM_SUMS_16BITS_IMPL(F16, vxc_short8,  vxc_half8)\n\
GROUP_NORM_SUMS_16BITS_IMPL(I16, vxc_short8,  vxc_short8)\n\
GROUP_NORM_SUMS_16BITS_IMPL(U16, vxc_ushort8, vxc_ushort8)\n\
\n\
#define GROUP_NORM_SUMS_16BITS_IMPL_2D(name, load_type, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sums_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int is2D) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 3; \\\n\
    int lidx = get_local_id(0); \\\n\
 \\\n\
    int2 coord = (int2)(gidx, get_global_id(1)); \\\n\
    load_type src; \\\n\
    src_type in_h; \\\n\
    float4 sumsqr = (float4)(0); \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    if(gidx < width) \\\n\
    { \\\n\
        VXC_ReadImage(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in_h, src, 16); \\\n\
        VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniSum_X_X2_8x2); \\\n\
        sumsqr.y = sumsqr.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * sumsqr.x; \\\n\
        sumsqr.x = sumsqr.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
 \\\n\
    lcl_sum[lidx] = sumsqr.x; \\\n\
    lcl_sqr[lidx] = sumsqr.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, get_global_id(1), 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        float sum = 0; \\\n\
        float sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 data = (float4)(sum, sqr, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
GROUP_NORM_SUMS_16BITS_IMPL_2D(F16, vxc_short8, vxc_half8)\n\
GROUP_NORM_SUMS_16BITS_IMPL_2D(I16, vxc_short8, vxc_short8)\n\
GROUP_NORM_SUMS_16BITS_IMPL_2D(U16, vxc_ushort8, vxc_ushort8)\n\
\n\
#define GROUP_NORM_16BITS_IMPL(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_array_t scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_para = (int4)((convert_int(gidx* rSpaceOrg) + convert_int(gidy * pStride)), gidz, 0, 0); \\\n\
    vxc_short8 src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    src_type in_h; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    _viv_asm(COPY, scale_h, src1, 16); \\\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
    copy_type outval; \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float alpha = input_scale * output_scale * scale_vari; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
    dst_type dst; \\\n\
 \\\n\
    _viv_asm(COPY, in_h, src0, 16); \\\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
 \\\n\
    float4 norm; \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_16BITS_IMPL(F16_F16toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
GROUP_NORM_16BITS_IMPL(F16_F16toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_IMPL(F16_F16toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
GROUP_NORM_16BITS_IMPL(F16_F16toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
GROUP_NORM_16BITS_IMPL(I16_F16toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_IMPL(I16_F16toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
\n\
#define GROUP_NORM_16BITS_IMPL_2D(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_array_t scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int2 coord = (int2)(get_global_id(0), gidz); \\\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0); \\\n\
    vxc_short8 src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    src_type in_h; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    _viv_asm(COPY, scale_h, src1, 16); \\\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
    copy_type outval; \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float alpha = output_scale * scale_vari; \\\n\
    bias_val = input_scale * (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
    dst_type dst; \\\n\
 \\\n\
    _viv_asm(COPY, in_h, src0, 16); \\\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    float4 norm; \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_16BITS_IMPL_2D(F16_F16toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
GROUP_NORM_16BITS_IMPL_2D(F16_F16toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_IMPL_2D(F16_F16toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
GROUP_NORM_16BITS_IMPL_2D(F16_F16toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
GROUP_NORM_16BITS_IMPL_2D(I16_F16toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_IMPL_2D(I16_F16toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
\n\
#define GROUP_NORM_16BITS_F32_IMPL(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_para = (int4)((convert_int(gidx * rSpaceOrg) + convert_int(gidy * pStride)), gidz, 0, 0); \\\n\
    vxc_short8 src0; \\\n\
    src_type in_h; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    scale_f = read_imagef(scale, coord_para.xy); \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
    copy_type outval; \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float alpha = input_scale * output_scale * scale_vari; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
    dst_type dst; \\\n\
 \\\n\
    _viv_asm(COPY, in_h, src0, 16); \\\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
 \\\n\
    float4 norm; \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_16BITS_F32_IMPL(F16_F32toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
GROUP_NORM_16BITS_F32_IMPL(F16_F32toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_F32_IMPL(F16_F32toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
GROUP_NORM_16BITS_F32_IMPL(F16_F32toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
GROUP_NORM_16BITS_F32_IMPL(I16_F32toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_F32_IMPL(I16_F32toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
GROUP_NORM_16BITS_F32_IMPL(U16_F32toU16, vxc_ushort8, vxc_ushort8, vxc_ushort8, int4)\n\
\n\
#define GROUP_NORM_16BITS_F32_IMPL_2D(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __read_only  image2d_t       meanVari, \\\n\
    __write_only image2d_array_t output, \\\n\
    float eps, int is2D, float rSpaceOrg, float pStride) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int2 coord = (int2)(get_global_id(0), gidz); \\\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0); \\\n\
    vxc_short8 src0; \\\n\
    src_type in_h; \\\n\
    float scale_vari, bias_val; \\\n\
    float4 bias_f, scale_f; \\\n\
 \\\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy); \\\n\
    bias_f = read_imagef(bias, coord_para.xy); \\\n\
    scale_f = read_imagef(scale, coord_para.xy); \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    scale_vari = scale_f.s0 * mean_vari.s1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
    copy_type outval; \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float alpha = input_scale * output_scale * scale_vari; \\\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp; \\\n\
    bias_val = bias_val - input_zp * alpha; \\\n\
    dst_type dst; \\\n\
 \\\n\
    _viv_asm(COPY, in_h, src0, 16); \\\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    float4 norm; \\\n\
    norm = alpha * tmpData0 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
    norm = alpha * tmpData1 + bias_val; \\\n\
    _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GROUP_NORM_16BITS_F32_IMPL_2D(F16_F32toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
GROUP_NORM_16BITS_F32_IMPL_2D(F16_F32toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_F32_IMPL_2D(F16_F32toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
GROUP_NORM_16BITS_F32_IMPL_2D(F16_F32toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
GROUP_NORM_16BITS_F32_IMPL_2D(I16_F32toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
GROUP_NORM_16BITS_F32_IMPL_2D(I16_F32toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
GROUP_NORM_16BITS_F32_IMPL_2D(U16_F32toU16, vxc_ushort8, vxc_ushort8, vxc_ushort8, int4)\n\
\n\
"; /* end of group_normalization_2_vx*/

static const char grucell_activation_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE     (1.44269502f)\n\
#define twoLogE  (2.88539004f)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hsigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvDatatoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float4 tensorScale;\n\
_viv_uniform float4 tensorZP;\n\
\n\
#define GRUCELL_ACTIVATION_SIGMOID_TANH(name0, name1, name2, name3, activater, \\\n\
        type00, type01, type10, type11, type20, type21, dst_type, conv_type, copy_type) \\\n\
__kernel void grucell_activation_##name0##_##name1##_##name2##_to_##name3##_##activater \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
    __write_only image2d_array_t hstate, \\\n\
                             int gate_activation, \\\n\
                             int candidate_activation \\\n\
    ) \\\n\
{ \\\n\
    type00 src00; \\\n\
    type01 src01; \\\n\
    type00 src10; \\\n\
    type01 src11; \\\n\
    type00 src20; \\\n\
    type01 src21; \\\n\
 \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    VXC_ReadImage(src00, input0, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src01, src00, 8); \\\n\
    VXC_ReadImage(src10, input1, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, src10, 8); \\\n\
    VXC_ReadImage(src20, input2, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src21, src20, 8); \\\n\
 \\\n\
    float4 zt, ht, ht_1; \\\n\
    VXC_DP4x4(zt, src01, src01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4); \\\n\
    VXC_DP4x4(ht, src11, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4); \\\n\
    VXC_DP4x4(ht_1, src21, src21, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4); \\\n\
 \\\n\
    zt = zt * tensorScale.xxxx - tensorZP.xxxx; \\\n\
    zt = activater(zt); \\\n\
 \\\n\
    ht = ht * tensorScale.yyyy - tensorZP.yyyy; \\\n\
    ht = tangentH(ht); \\\n\
 \\\n\
    ht_1 = ht_1 * tensorScale.zzzz - tensorZP.zzzz; \\\n\
 \\\n\
    ht = ht - zt * ht; \\\n\
    ht = zt * ht_1 + ht; \\\n\
 \\\n\
    ht = ht  * tensorScale.wwww + tensorZP.wwww; \\\n\
    conv_type dst0; \\\n\
    dst_type dst1; \\\n\
    copy_type dst; \\\n\
 \\\n\
    _viv_asm(CONV_RTE, dst0, ht); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
#define UCHAR8  vxc_uchar8\n\
#define SHORT8  vxc_short8\n\
#define HALF8   vxc_half8\n\
\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(U8,  U8,  U8,  U8, sigmoid,\n\
                                UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, int4,  UCHAR8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, F16, sigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  HALF8,  half4, SHORT8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, U8, sigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  UCHAR8, int4,  UCHAR8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(U8,  U8,  U8,  U8, hsigmoid,\n\
                                UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, int4,  UCHAR8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, F16, hsigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  HALF8,  half4, SHORT8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, U8, hsigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  UCHAR8, int4,  UCHAR8)\n\
\n\
#undef UCHAR8\n\
#undef SHORT8\n\
#undef HALF8\n\
\n\
#define GRUCELL_ACTIVATION_SIGMOID_TANH_BF16(activater) \\\n\
__kernel void grucell_activation_BF16_BF16_BF16_to_BF16_##activater \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
    __write_only image2d_array_t hstate, \\\n\
                             int gate_activation, \\\n\
                             int candidate_activation \\\n\
    ) \\\n\
{ \\\n\
    vxc_short8 src00, src10, src20, data0, data1; \\\n\
    float4 src01, src11, src21; \\\n\
 \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    VXC_ReadImage(src00, input0, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, input1, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src20, input2, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, src00, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src01, data0, 16); \\\n\
    VXC_DP2x8(data1, src10, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, src20, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src21, data0, 16); \\\n\
 \\\n\
    src01 = src01 * tensorScale.xxxx - tensorZP.xxxx; \\\n\
    src01 = activater(src01); \\\n\
 \\\n\
    src11 = src11 * tensorScale.yyyy - tensorZP.yyyy; \\\n\
    src11 = tangentH(src11); \\\n\
 \\\n\
    src21 = src21 * tensorScale.zzzz - tensorZP.zzzz; \\\n\
 \\\n\
    src11 = src11 - src01 * src11; \\\n\
    src11 = src01 * src21 + src11; \\\n\
 \\\n\
    src11 = src11  * tensorScale.wwww + tensorZP.wwww; \\\n\
    _viv_asm(COPY, src00, src11, 16); \\\n\
    VXC_DP2x8(data0, src00, src00, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate, coord, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_ACTIVATION_SIGMOID_TANH_BF16(sigmoid)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH_BF16(hsigmoid)\n\
"; /* end of grucell_activation_vx*/

static const char grucell_activation_sma_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniA_Minus_B_2x8;\n\
_viv_uniform VXC_512Bits uniA_Times_B_2x8;\n\
_viv_uniform VXC_512Bits uniA_Plus_B_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void grucell_activation_sma_F16_F16_F16toF16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  h_status\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_half8   src0, src1, src2, minus, dst;\n\
    vxc_ushort8 vec0, vec1, vec2;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage2DArray(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(minus, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Minus_B_2x8);\n\
    VXC_DP2x8(dst, minus, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(h_status, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_sma_F16_F16_F16toF16_2D\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  h_status\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_half8   src0, src1, src2, minus, dst;\n\
    vxc_ushort8 vec0, vec1, vec2;\n\
\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(minus, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Minus_B_2x8);\n\
    VXC_DP2x8(dst, minus, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(h_status, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_sma_BF16_BF16_BF16toBF16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  h_status\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0, src00, src1, src11, src2, src22, minus, minus1, dst, dst1;\n\
    vxc_ushort8 vec0, vec1, vec2, data0, data1;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(data0, vec0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(data1, vec0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src0, data0, 16);\n\
    _viv_asm(COPY, src00, data1, 16);\n\
    VXC_DP2x8(data0, vec1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(data1, vec1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src1, data0, 16);\n\
    _viv_asm(COPY, src11, data1, 16);\n\
    VXC_DP2x8(data0, vec2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(data1, vec2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src2, data0, 16);\n\
    _viv_asm(COPY, src22, data1, 16);\n\
\n\
    minus = src0 - src1;\n\
    minus1 = src00 - src11;\n\
\n\
    dst = minus * src2 + src1;\n\
    dst1 = minus1 * src22 + src11;\n\
    _viv_asm(COPY, vec0, dst, 16);\n\
    _viv_asm(COPY, vec1, dst1, 16);\n\
    VXC_DP2x8(data0, vec0, vec1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, data0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(h_status, coord, data0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_sma_BF16_BF16_BF16toBF16_2D\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  h_status\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0, src00, src1, src11, src2, src22, minus, minus1, dst, dst1;\n\
    vxc_ushort8 vec0, vec1, vec2, data0, data1;\n\
\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(data0, vec0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(data1, vec0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src0, data0, 16);\n\
    _viv_asm(COPY, src00, data1, 16);\n\
    VXC_DP2x8(data0, vec1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(data1, vec1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src1, data0, 16);\n\
    _viv_asm(COPY, src11, data1, 16);\n\
    VXC_DP2x8(data0, vec2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(data1, vec2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src2, data0, 16);\n\
    _viv_asm(COPY, src22, data1, 16);\n\
\n\
    minus = src0 - src1;\n\
    minus1 = src00 - src11;\n\
\n\
    dst = minus * src2 + src1;\n\
    dst1 = minus1 * src22 + src11;\n\
    _viv_asm(COPY, vec0, dst, 16);\n\
    _viv_asm(COPY, vec1, dst1, 16);\n\
    VXC_DP2x8(data0, vec0, vec1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord, data0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(h_status, coord, data0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of grucell_activation_sma_vx*/

static const char grucell_activation_z_h_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float output_scale1;\n\
_viv_uniform float output_zp1;\n\
\n\
float4 sigmoid_func(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1.0f + exp2(x);\n\
    return 1.0f / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tanh_func(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1.0f / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniF16PlusF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniF16PlusF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define GRUCELL_F16_F16TOF16(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_F16_F16toF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, v3, 16); \\\n\
 \\\n\
    float4 h; \\\n\
    VXC_DP4x4(h, src2, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = act_func(z); \\\n\
    h = tanh_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half4 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_F16_F16TOF16(SIGMOID,  sigmoid_func)\n\
GRUCELL_F16_F16TOF16(HSIGMOID, hard_sigmoid)\n\
\n\
_viv_uniform float hstate_in_scale;\n\
_viv_uniform float hstate_in_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
#define GRUCELL_QNT_F16TO_QNT(name0, name1, act_name, act_func, src0_type, dst_type) \\\n\
__kernel void grucell_activation_z_h_##name0##_F16to##name1##_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type src3; \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src4, src5, src6; \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(src3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 h; \\\n\
    VXC_DP4x4(h, src2, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = act_func(z); \\\n\
    h = tanh_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    h_tm = h_tm * hstate_in_scale + hstate_in_tail; \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    float4 out0 = result * output_scale + output_zp; \\\n\
    float4 out1 = result * output_scale1 + output_zp1; \\\n\
    int4 dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, out0); \\\n\
    _viv_asm(CONV_RTE, dst1, out1); \\\n\
    dst_type dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_QNT_F16TO_QNT(U8,  U8,  SIGMOID,  sigmoid_func, vxc_uchar8, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_QNT(I8,  I8,  SIGMOID,  sigmoid_func, vxc_char8,  vxc_char8)\n\
GRUCELL_QNT_F16TO_QNT(I16, I16, SIGMOID,  sigmoid_func, vxc_short8, vxc_short8)\n\
GRUCELL_QNT_F16TO_QNT(U8,  U8,  HSIGMOID, hard_sigmoid, vxc_uchar8, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_QNT(I8,  I8,  HSIGMOID, hard_sigmoid, vxc_char8,  vxc_char8)\n\
GRUCELL_QNT_F16TO_QNT(I16, I16, HSIGMOID, hard_sigmoid, vxc_short8, vxc_short8)\n\
\n\
#define GRUCELL_BF16(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_BF16_BF16toBF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6, data0, data1; \\\n\
    float4 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, v4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src4, data1, 16); \\\n\
    VXC_DP2x8(data0, v5, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src5, data0, 16); \\\n\
    VXC_DP2x8(data1, v6, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src6, data1, 16); \\\n\
    VXC_DP2x8(data0, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
 \\\n\
    float4 h = src2 + src4; \\\n\
    float4 z = src5 + src6; \\\n\
    h = tanh_func(h); \\\n\
    z = act_func(z); \\\n\
    float4 result = (1 - z) * h + z * src3; \\\n\
    _viv_asm(COPY, v0, result, 16); \\\n\
    VXC_DP2x8(data0, v0, v0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_BF16(SIGMOID,  sigmoid_func)\n\
GRUCELL_BF16(HSIGMOID, hard_sigmoid)\n\
"; /* end of grucell_activation_z_h_vx*/

static const char grucell_cdnn_activation_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE     (1.44269502f)\n\
#define twoLogE  (2.88539004f)\n\
\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvDatatoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uiF16AddF16_4x4;\n\
\n\
__kernel void grucell_activation_cdnn_sep_F16_F16_F16_to_F16_NC\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord);\n\
    r3 = read_imagef(cond_r, coord);\n\
\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord);\n\
    z3 = read_imagef(cond_z, coord);\n\
\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord);\n\
    c3 = read_imagef(cond_c, coord);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2 + r3;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2 + z3;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_F16_F16_F16_to_F16_CN\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2.xxxx + r3.xxxx;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_F16_F16_F16_to_F16_CN_FULL\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2.xxxx + r3.xxxx;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
\n\
__kernel void grucell_activation_cdnn_F16_F16_F16_to_F16\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_rzc,\n\
    __read_only  image2d_array_t recur_rzc,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(1) * 3, get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord.xy);\n\
    r3 = read_imagef(cond_r, coord.xy);\n\
\n\
    VXC_ReadImage(s2, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord.xy);\n\
    z3 = read_imagef(cond_z, coord.xy);\n\
\n\
    VXC_ReadImage(s4, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord.xy);\n\
    c3 = read_imagef(cond_c, coord.xy);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2 + r3;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2 + z3;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of grucell_cdnn_activation_vx*/

static const char grucell_cdnn_activation_bf16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE     (1.44269502f)\n\
#define twoLogE  (2.88539004f)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_BF16_BF16_BF16_to_BF16_NC(\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
        int gate_activation, int candidate_activation, int batch_first)\n\
{\n\
    vxc_ushort8 s0, s1, s2, s3, s4, s5, s7, data0, data1;\n\
    float4 r0, r1, z0, z1, c0, c1, state;\n\
    float4 r, r2, r3, z, z2, z3, c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord);\n\
    r3 = read_imagef(cond_r, coord);\n\
    z2 = read_imagef(bias_z, coord);\n\
    z3 = read_imagef(cond_z, coord);\n\
    c2 = read_imagef(bias_c, coord);\n\
    c3 = read_imagef(cond_c, coord);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(data0, s0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r0, data0, 16);\n\
    VXC_DP2x8(data1, s1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r1, data1, 16);\n\
    VXC_DP2x8(data0, s2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z0, data0, 16);\n\
    VXC_DP2x8(data1, s3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z1, data1, 16);\n\
    VXC_DP2x8(data0, s4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c0, data0, 16);\n\
    VXC_DP2x8(data1, s5, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c1, data1, 16);\n\
    VXC_DP2x8(data0, s7, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, state, data0, 16);\n\
    r = r0 + r1 + r2 + r3;\n\
    z = z0 + z1 + z2 + z3;\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    c = c0 + c1 * r + c;\n\
    c = tangentH(c);\n\
\n\
    state = z * (state - c) + c;\n\
    _viv_asm(COPY, s0, state, 16);\n\
    VXC_DP2x8(data0, s0, s0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
    VXC_WriteImage(output, coord, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_BF16_BF16_BF16_to_BF16_CN(\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
    int gate_activation, int candidate_activation, int batch_first)\n\
{\n\
    vxc_ushort8 s0, s1, s2, s3, s4, s5, s7, data0, data1;\n\
    float4 r0, r1, z0, z1, c0, c1, state;\n\
    float4 r, r2, r3, z, z2, z3, c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(data0, s0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r0, data0, 16);\n\
    VXC_DP2x8(data1, s1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r1, data1, 16);\n\
    VXC_DP2x8(data0, s2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z0, data0, 16);\n\
    VXC_DP2x8(data1, s3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z1, data1, 16);\n\
    VXC_DP2x8(data0, s4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c0, data0, 16);\n\
    VXC_DP2x8(data1, s5, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c1, data1, 16);\n\
    VXC_DP2x8(data0, s7, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, state, data0, 16);\n\
\n\
    r = r0 + r1 + r2.xxxx + r3.xxxx;\n\
    z = z0 + z1 + z2.xxxx + z3.xxxx;\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    c = c0 + c1 * r + c;\n\
    c = tangentH(c);\n\
    state = z * (state - c) + c;\n\
\n\
    _viv_asm(COPY, s0, state, 16);\n\
    VXC_DP2x8(data0, s0, s0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
    VXC_WriteImage(output, coord.yx, data0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, data0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, data0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, data0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, data0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, data0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, data0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, data0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_BF16_BF16_BF16_to_BF16_CN_FULL(\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
    int gate_activation, int candidate_activation, int batch_first)\n\
{\n\
    vxc_ushort8 s0, s1, s2, s3, s4, s5, s7, data0, data1;\n\
    float4 r0, r1, z0, z1, c0, c1, state;\n\
    float4 r, r2, r3, z, z2, z3, c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(data0, s0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r0, data0, 16);\n\
    VXC_DP2x8(data1, s1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r1, data1, 16);\n\
    VXC_DP2x8(data0, s2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z0, data0, 16);\n\
    VXC_DP2x8(data1, s3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z1, data1, 16);\n\
    VXC_DP2x8(data0, s4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c0, data0, 16);\n\
    VXC_DP2x8(data1, s5, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c1, data1, 16);\n\
    VXC_DP2x8(data0, s7, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, state, data0, 16);\n\
\n\
    r = r0 + r1 + r2.xxxx + r3.xxxx;\n\
    z = z0 + z1 + z2.xxxx + z3.xxxx;\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    c = c0 + c1 * r + c;\n\
    c = tangentH(c);\n\
    state = z * (state - c) + c;\n\
\n\
    _viv_asm(COPY, s0, state, 16);\n\
    VXC_DP2x8(data0, s0, s0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
\n\
__kernel void grucell_activation_cdnn_BF16_BF16_BF16_to_BF16(\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_rzc,\n\
    __read_only  image2d_array_t recur_rzc,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
        int gate_activation, int candidate_activation, int batch_first)\n\
{\n\
    vxc_ushort8 s0, s1, s2, s3, s4, s5, s7, data0, data1;\n\
    float4 r0, r1, z0, z1, c0, c1, state;\n\
    float4 r, r2, r3, z, z2, z3, c, c2, c3;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(1) * 3, get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s1, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s2, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s3, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s4, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s5, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r2 = read_imagef(bias_r, coord.xy);\n\
    r3 = read_imagef(cond_r, coord.xy);\n\
    z2 = read_imagef(bias_z, coord.xy);\n\
    z3 = read_imagef(cond_z, coord.xy);\n\
    c2 = read_imagef(bias_c, coord.xy);\n\
    c3 = read_imagef(cond_c, coord.xy);\n\
\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(data0, s0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r0, data0, 16);\n\
    VXC_DP2x8(data1, s1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, r1, data1, 16);\n\
    VXC_DP2x8(data0, s2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z0, data0, 16);\n\
    VXC_DP2x8(data1, s3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, z1, data1, 16);\n\
    VXC_DP2x8(data0, s4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c0, data0, 16);\n\
    VXC_DP2x8(data1, s5, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, c1, data1, 16);\n\
    VXC_DP2x8(data0, s7, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, state, data0, 16);\n\
\n\
    r = r0 + r1 + r2 + r3;\n\
    z = z0 + z1 + z2 + z3;\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    c = c0 + c1 * r + c;\n\
    c = tangentH(c);\n\
    state = z * (state - c) + c;\n\
\n\
    _viv_asm(COPY, s0, state, 16);\n\
    VXC_DP2x8(data0, s0, s0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord.xy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.xy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of grucell_cdnn_activation_bf16_vx*/

static const char grucell_cdnn_activation_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE     (1.44269502f)\n\
#define twoLogE  (2.88539004f)\n\
\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvDatatoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float input_r_scale;\n\
_viv_uniform float input_r_tail;\n\
_viv_uniform float recur_r_scale;\n\
_viv_uniform float recur_r_tail;\n\
_viv_uniform float input_z_scale;\n\
_viv_uniform float input_z_tail;\n\
_viv_uniform float recur_z_scale;\n\
_viv_uniform float recur_z_tail;\n\
_viv_uniform float input_c_scale;\n\
_viv_uniform float input_c_tail;\n\
_viv_uniform float recur_c_scale;\n\
_viv_uniform float recur_c_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
__kernel void grucell_activation_cdnn_sep_U8_U8_U8_to_U8_NC\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8  r00, r01;\n\
    vxc_uchar8  z0, z1;\n\
    vxc_uchar8  c0, c1;\n\
    float4      r, r0, r1, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord);\n\
    r3 = read_imagef(cond_r, coord);\n\
\n\
    VXC_ReadImage(z0, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord);\n\
    z3 = read_imagef(cond_z, coord);\n\
\n\
    VXC_ReadImage(c0, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord);\n\
    c3 = read_imagef(cond_c, coord);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2 + r3;\n\
\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_z_scale + input_z_tail;\n\
    r1 = r1 * recur_z_scale + recur_z_tail;\n\
    z = r0 + r1 + z2 + z3;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_U8_U8_U8_to_U8_CN\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8   r00, r01;\n\
    vxc_uchar8   z0, z1;\n\
    vxc_uchar8   c0, c1;\n\
    float4      r, r2, r3, r0, r1;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(z0, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(c0, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2.xxxx + r3.xxxx;\n\
\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_z_scale + input_z_tail;\n\
    r1 = r1 * recur_z_scale + recur_z_tail;\n\
    z = r0 + r1 + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord.yx, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(h, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(h, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(h, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_U8_U8_U8_to_U8_CN_FULL\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8   r00, r01;\n\
    vxc_uchar8   z0, z1;\n\
    vxc_uchar8   c0, c1;\n\
    float4      r, r2, r3, r0, r1;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(z0, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(c0, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2.xxxx + r3.xxxx;\n\
\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_z_scale + input_z_tail;\n\
    r1 = r1 * recur_z_scale + recur_z_tail;\n\
    z = r0 + r1 + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_U8_U8_U8_to_U8\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_rzc,\n\
    __read_only  image2d_array_t recur_rzc,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8   r00, r01;\n\
    vxc_uchar8   z0, z1;\n\
    vxc_uchar8   c0, c1;\n\
    float4      r, r0, r1, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(1) * 3, get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_rzc, coord.xz, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_rzc, coord.xz, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.xy);\n\
    r3 = read_imagef(cond_r, coord.xy);\n\
\n\
    VXC_ReadImage(z0, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord.xy);\n\
    z3 = read_imagef(cond_z, coord.xy);\n\
\n\
    VXC_ReadImage(c0, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord.xy);\n\
    c3 = read_imagef(cond_c, coord.xy);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2 + r3;\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * input_r_scale + recur_r_tail;\n\
    z = r0 + r1 + z2 + z3;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of grucell_cdnn_activation_u8_vx*/

static const char grucell_h_times_activation_r_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
float4 sigmoid_func(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1.0f + exp2(x);\n\
    return 1.0f / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniF16PlusF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniF16PlusF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define GRUCELL_F16_F16TOF16(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_F16_F16toF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __write_only image2d_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, v3, 16); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = act_func(r); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 result = r * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half4 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_F16_F16TOF16(SIGMOID,  sigmoid_func)\n\
GRUCELL_F16_F16TOF16(HSIGMOID, hard_sigmoid)\n\
\n\
_viv_uniform float hstate_in_scale;\n\
_viv_uniform float hstate_in_tail;\n\
#define GRUCELL_QNT_F16TO_F16(name0, act_name, act_func, src0_type) \\\n\
__kernel void grucell_h_times_activation_r_##name0##_F16toF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __write_only image2d_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type src3; \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(src3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = act_func(r); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    h_tm = h_tm * hstate_in_scale + hstate_in_tail; \\\n\
    float4 result = r * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half8 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_QNT_F16TO_F16(U8,  SIGMOID,  sigmoid_func, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_F16(I8,  SIGMOID,  sigmoid_func, vxc_char8)\n\
GRUCELL_QNT_F16TO_F16(I16, SIGMOID,  sigmoid_func, vxc_short8)\n\
GRUCELL_QNT_F16TO_F16(U8,  HSIGMOID, hard_sigmoid, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_F16(I8,  HSIGMOID, hard_sigmoid, vxc_char8)\n\
GRUCELL_QNT_F16TO_F16(I16, HSIGMOID, hard_sigmoid, vxc_short8)\n\
\n\
#define GRUCELL_BF16(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_BF16_BF16toBF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __write_only image2d_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, data0, data1; \\\n\
    float4 src0, src1, src2, src3; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_DP2x8(data1, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
    VXC_DP2x8(data0, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
 \\\n\
    float4 r; \\\n\
    r = src0 + src1; \\\n\
    r = act_func(r); \\\n\
    float4 result = r * src3; \\\n\
    _viv_asm(COPY, v0, result, 16); \\\n\
    VXC_DP2x8(data0, v0, v0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_BF16(SIGMOID,  sigmoid_func)\n\
GRUCELL_BF16(HSIGMOID, hard_sigmoid)\n\
"; /* end of grucell_h_times_activation_r_vx*/

static const char grucell_reset_after_activation_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
float4 sigmoid_func(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1.0f + exp2(x);\n\
    return 1.0f / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tanh_func(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1.0f / x;\n\
    return 2 * x - 1;\n\
}\n\
float4 relu_func(float4 x)\n\
{\n\
    x = x > 0 ? x : 0;\n\
    return x;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniF16PlusF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniF16PlusF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define GRUCELL_F16_F16TOF16(act_name, act_func, rec_act_name, rec_act_func) \\\n\
__kernel void grucell_reset_after_activation_F16_F16toF16_##act_name##_##rec_act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, v3, 16); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = rec_act_func(r); \\\n\
    float4 h0, h1; \\\n\
    VXC_DP4x4(h1, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    VXC_DP4x4(h0, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 h = h0 + r * h1; \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = rec_act_func(z); \\\n\
    h = act_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half4 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_F16_F16TOF16(TANH,    tanh_func,    SIGMOID, sigmoid_func)\n\
GRUCELL_F16_F16TOF16(SIGMOID, sigmoid_func, SIGMOID, sigmoid_func)\n\
GRUCELL_F16_F16TOF16(RELU,    relu_func,    SIGMOID, sigmoid_func)\n\
\n\
\n\
_viv_uniform float hstate_in_scale;\n\
_viv_uniform float hstate_in_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
#define GRUCELL_QNT_F16TO_QNT(name, act_func, rec_act_func, src0_type, dst_type) \\\n\
__kernel void grucell_reset_after_activation_##name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type src3; \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(src3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = rec_act_func(r); \\\n\
    float4 h0, h1; \\\n\
    VXC_DP4x4(h1, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    VXC_DP4x4(h0, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 h = h0 + r * h1; \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = rec_act_func(z); \\\n\
    h = act_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    h_tm = h_tm * hstate_in_scale + hstate_in_tail; \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    result = result * output_scale + output_zp; \\\n\
    int4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    dst_type dst; \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_QNT_F16TO_QNT(U8_F16toU8_TANH_SIGMOID,      tanh_func,    sigmoid_func, vxc_uchar8, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_QNT(I8_F16toI8_TANH_SIGMOID,      tanh_func,    sigmoid_func, vxc_char8,  vxc_char8)\n\
GRUCELL_QNT_F16TO_QNT(I16_F16toI16_TANH_SIGMOID,    tanh_func,    sigmoid_func, vxc_short8, vxc_short8)\n\
GRUCELL_QNT_F16TO_QNT(U8_F16toU8_SIGMOID_SIGMOID,   sigmoid_func, sigmoid_func, vxc_uchar8, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_QNT(I8_F16toI8_SIGMOID_SIGMOID,   sigmoid_func, sigmoid_func, vxc_char8,  vxc_char8)\n\
GRUCELL_QNT_F16TO_QNT(I16_F16toI16_SIGMOID_SIGMOID, sigmoid_func, sigmoid_func, vxc_short8, vxc_short8)\n\
GRUCELL_QNT_F16TO_QNT(U8_F16toU8_RELU_SIGMOID,      relu_func,    sigmoid_func, vxc_uchar8, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_QNT(I8_F16toI8_RELU_SIGMOID,      relu_func,    sigmoid_func, vxc_char8,  vxc_char8)\n\
GRUCELL_QNT_F16TO_QNT(I16_F16toI16_RELU_SIGMOID,    relu_func,    sigmoid_func, vxc_short8, vxc_short8)\n\
\n\
\n\
#define GRUCELL_BF16(act_name, act_func, rec_act_name, rec_act_func) \\\n\
__kernel void grucell_reset_after_activation_BF16_BF16toBF16_##act_name##_##rec_act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6, data0, data1; \\\n\
    float4 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_DP2x8(data1, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
    VXC_DP2x8(data0, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, v4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src4, data1, 16); \\\n\
    VXC_DP2x8(data0, v5, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src5, data0, 16); \\\n\
    VXC_DP2x8(data1, v6, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src6, data1, 16); \\\n\
    VXC_DP2x8(data0, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                         uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
 \\\n\
    float4 r; \\\n\
    r = src0 + src1; \\\n\
    r = rec_act_func(r); \\\n\
    float4 h = src4 + r * src2; \\\n\
    float4 z = src5 + src6; \\\n\
    h = act_func(h); \\\n\
    z = rec_act_func(z); \\\n\
    float4 result = (1 - z) * h + z * src3; \\\n\
    _viv_asm(COPY, v0, result, 16); \\\n\
    VXC_DP2x8(data0, v0, v0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_BF16(TANH,    tanh_func,    SIGMOID, sigmoid_func)\n\
GRUCELL_BF16(SIGMOID, sigmoid_func, SIGMOID, sigmoid_func)\n\
GRUCELL_BF16(RELU,    relu_func,    SIGMOID, sigmoid_func)\n\
"; /* end of grucell_reset_after_activation_vx*/

static const char hswish_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define HSWISH_PROCESS(read_fun, write_fun, src_type, src_copy_type, convert_type, dst_type, dst_copy_type, \\\n\
                     INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
    float4 vecA, vecB, vecC, vecD, vecE, vecDstA, vecDstB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * INSCALE + INTAIL; \\\n\
    vecB = vecB * INSCALE + INTAIL; \\\n\
    vecC = vecA + 3.0f; \\\n\
    vecD = vecB + 3.0f; \\\n\
    vecE = 6.0f; \\\n\
    _viv_asm(CLAMP0MAX, vecDstA, vecC, vecE); \\\n\
    _viv_asm(CLAMP0MAX, vecDstB, vecD, vecE); \\\n\
    vecA = vecA * vecDstA; \\\n\
    vecB = vecB * vecDstB; \\\n\
    vecA = vecA / 6.0f; \\\n\
    vecB = vecB / 6.0f; \\\n\
    vecA = vecA * OUTSCALE + OUTZP; \\\n\
    vecB = vecB * OUTSCALE + OUTZP; \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define HSWISH_FUNC(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, \\\n\
                   dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void hswish_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    HSWISH_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, src_type, \\\n\
                 src_copy_type, convert_type, dst_type, dst_copy_type, \\\n\
                 INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
HSWISH_FUNC(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
HSWISH_FUNC(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
HSWISH_FUNC(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
HSWISH_FUNC(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
HSWISH_FUNC(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
HSWISH_FUNC(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, \\\n\
            inputScale, inputTail, outputScale, outputZP)\n\
HSWISH_FUNC(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
HSWISH_FUNC(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
#define HSWISH_FUNC_2D(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, \\\n\
                      dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void hswish_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    HSWISH_PROCESS(VXC_ReadImage, VXC_WriteImage, src_type, src_copy_type, convert_type, dst_type, \\\n\
                   dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
HSWISH_FUNC_2D(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
HSWISH_FUNC_2D(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
HSWISH_FUNC_2D(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
HSWISH_FUNC_2D(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, inputScale, \\\n\
               inputTail, outputScale, outputZP)\n\
HSWISH_FUNC_2D(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
HSWISH_FUNC_2D(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define HSWISH_BF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    float4 vecA, vecB, vecC, vecD, vecE, vecDstA, vecDstB; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecC = vecA + 3.0f; \\\n\
    vecD = vecB + 3.0f; \\\n\
    vecE = 6.0f; \\\n\
    _viv_asm(CLAMP0MAX, vecDstA, vecC, vecE); \\\n\
    _viv_asm(CLAMP0MAX, vecDstB, vecD, vecE); \\\n\
    vecA = vecA * vecDstA; \\\n\
    vecB = vecB * vecDstB; \\\n\
    vecA = vecA / 6.0f; \\\n\
    vecB = vecB / 6.0f; \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void hswish_BF16toBF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_BF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray);\n\
}\n\
\n\
__kernel void hswish_BF16toBF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_BF16_PROCESS(VXC_ReadImage, VXC_WriteImage);\n\
}\n\
"; /* end of hswish_vx*/

static const char instance_normalization_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniSum_X_X2_16x2;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_scale2;\n\
_viv_uniform float sum_x_tail;\n\
_viv_uniform float sum_x2_tail0;\n\
_viv_uniform float sum_x2_tail1;\n\
\n\
_viv_uniform VXC_512Bits uniSumX_16x1;\n\
_viv_uniform VXC_512Bits uniSumX2_16x1;\n\
\n\
#define INSTANCE_NORM_SUMS_8BITS_IMPL(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_sums_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int height) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz); \\\n\
    src_type src0; \\\n\
    float2 sums_f32 = 0; \\\n\
    int2 sums = 0, sum_x_x2; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
    if(gidx < width) \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src0, input, coord, 0, \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
            VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
            sums = sums + sum_x_x2; \\\n\
        } \\\n\
        sums_f32 = convert_float2(sums); \\\n\
        sums_f32.y = sums_f32.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * sums_f32.x; \\\n\
        sums_f32.x = sums_f32.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
    lcl_sum[lidx] = sums_f32.x; \\\n\
    lcl_sqr[lidx] = sums_f32.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        float sum = 0, sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 data = (float4)(sum, sqr, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_SUMS_8BITS_IMPL(U8, vxc_uchar16)\n\
INSTANCE_NORM_SUMS_8BITS_IMPL(I8, vxc_char16)\n\
\n\
#define INSTANCE_NORM_SUMS_8BITS_IMPL_2D(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_sums_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int height) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int gidy = gidz * height; \\\n\
 \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    src_type src0; \\\n\
    float2 sums_f32 = 0; \\\n\
    int2 sums = 0, sum_x_x2; \\\n\
    int endH = gidy + height; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    if (gidx < width) \\\n\
    { \\\n\
        for(; coord.y < endH;) \\\n\
        { \\\n\
            VXC_ReadImage(src0, input, coord, 0,  \\\n\
                             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
            VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
            sums = sums + sum_x_x2; \\\n\
        } \\\n\
        sums_f32 = convert_float2(sums); \\\n\
        sums_f32.y = sums_f32.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * sums_f32.x; \\\n\
        sums_f32.x = sums_f32.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
    lcl_sum[lidx] = sums_f32.x; \\\n\
    lcl_sqr[lidx] = sums_f32.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        float sum = 0, sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 data = (float4)(sum, sqr, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_SUMS_8BITS_IMPL_2D(U8, vxc_uchar16)\n\
INSTANCE_NORM_SUMS_8BITS_IMPL_2D(I8, vxc_char16)\n\
\n\
__kernel void instance_norm_means\n\
(\n\
    __read_only  image2d_t sums,\n\
    __read_only  image2d_t bias,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t means,\n\
                 float     eps,\n\
                 float     in_time_out_scale,\n\
                 float     input_zp,\n\
                 float     output_scale,\n\
                 float     output_zp,\n\
                 float     inv_multiplier,\n\
                 int       group_num\n\
)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    Image sums_img = create_image_from_image2d(sums, 4);\n\
    float4 *sums_ptr = (float4 *)get_image_ptr_from_coord(sums_img, coord);\n\
\n\
    float alpha = read_imagef(scale, coord).x;\n\
    float beta = read_imagef(bias, coord).x;\n\
\n\
    float4 mean_var = sums_ptr[0];\n\
    for(int i = 1; i < group_num;)\n\
    {\n\
        mean_var += sums_ptr[i];\n\
        i ++;\n\
    }\n\
\n\
    mean_var *= inv_multiplier;\n\
    mean_var.s1 = mean_var.s1 - mean_var.s0 * mean_var.s0 + eps;\n\
    mean_var.s1 = rsqrt(mean_var.s1);\n\
\n\
    alpha = alpha * mean_var.y;\n\
\n\
    float4 dst;\n\
    dst.x = in_time_out_scale * alpha;\n\
    beta = (beta - alpha * mean_var.x) * output_scale + output_zp;\n\
    dst.y = beta - input_zp * dst.x;\n\
\n\
    Image means_img = create_image_from_image2d(means, 4);\n\
    float4 *means_ptr = (float4 *)get_image_ptr_from_coord(means_img, coord);\n\
    means_ptr[0] = dst.xyxy;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_2_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_3_4x4;\n\
#define INSTANCE_NORM_8BITS_IMPL(name, src_type, dst_type) \\\n\
__kernel void instance_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       means, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int height) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_para = (int2)(0, gidz); \\\n\
    src_type src0; \\\n\
    dst_type dst; \\\n\
    float4 coef; \\\n\
 \\\n\
    coef = read_imagef(means, coord_para); \\\n\
    int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, 0, \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.y ++; \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = tmpData0 * coef.x + coef.y; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * coef.x + coef.y; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    norm = tmpData2 * coef.x + coef.y; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * coef.x + coef.y; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_8BITS_IMPL(U8_F32toU8, vxc_uchar16, vxc_uchar16)\n\
INSTANCE_NORM_8BITS_IMPL(I8_F32toI8, vxc_char16,  vxc_char16)\n\
\n\
#define INSTANCE_NORM_8BITS_IMPL_2D(name, src_type, dst_type) \\\n\
__kernel void instance_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       means, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int height) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int gidy = gidz * height; \\\n\
    int4 coord; \\\n\
    int2 coord_para = (int2)(0, gidz); \\\n\
    int endH = gidy + height; \\\n\
    src_type src0; \\\n\
    dst_type dst; \\\n\
    float4 coef; \\\n\
 \\\n\
    coef = read_imagef(means, coord_para); \\\n\
    int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
 \\\n\
    coord = (int4)(get_global_id(0), gidy, gidy - 1, gidy - 1); \\\n\
 \\\n\
    for(; coord.y < endH; ) \\\n\
    { \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.yz++; \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = tmpData0 * coef.x + coef.y; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * coef.x + coef.y; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    norm = tmpData2 * coef.x + coef.y; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * coef.x + coef.y; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_8BITS_IMPL_2D(U8_F32toU8, vxc_uchar16, vxc_uchar16)\n\
INSTANCE_NORM_8BITS_IMPL_2D(I8_F32toI8, vxc_char16,  vxc_char16)"; /* end of instance_normalization_0_vx*/

static const char instance_normalization_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_2_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_3_4x4;\n\
\n\
#define INSTANCE_NORM_8_TO_F16_IMPL(name, src_type) \\\n\
__kernel void instance_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       means, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int height) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_para = (int4)(0, gidz, 0, 0); \\\n\
    src_type src0; \\\n\
    vxc_short8 outval; \\\n\
    vxc_half8 dst; \\\n\
    float4 coef; \\\n\
 \\\n\
    coef = read_imagef(means, coord_para.xy); \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
 \\\n\
    coord_para = coord; \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_para.z, baseAddr); \\\n\
    for(coord.y = 0; coord.y < height;) \\\n\
    { \\\n\
    VXC_OP4(img_load_3d, src0, input, coord, 0, \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_para.xy = coord.xy; \\\n\
    coord.y++; \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = tmpData0 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = tmpData1 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_para, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_para.x += 8; \\\n\
    norm = tmpData2 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = tmpData3 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_para, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_8_TO_F16_IMPL(U8_F32toF16, vxc_uchar16)\n\
INSTANCE_NORM_8_TO_F16_IMPL(I8_F32toF16, vxc_char16)\n\
\n\
#define INSTANCE_NORM_8_TO_F16_IMPL_2D(name, src_type) \\\n\
__kernel void instance_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       means, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int height) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int gidy = gidz * height; \\\n\
    int4 coord = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    int4 coord_para = (int4)(0, gidz, 0, 0); \\\n\
    int endH = gidy + height; \\\n\
    src_type src0; \\\n\
    vxc_short8 outval; \\\n\
    vxc_half8 dst; \\\n\
    float4 coef; \\\n\
 \\\n\
    coef = read_imagef(means, coord_para.xy); \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    for(; coord.y < endH;) \\\n\
    { \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_para = coord; \\\n\
    coord.y++; \\\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_2_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_3_4x4); \\\n\
    norm = tmpData0 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = tmpData1 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord_para.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_para.x += 8; \\\n\
    norm = tmpData2 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = tmpData3 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord_para.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_8_TO_F16_IMPL_2D(U8_F32toF16, vxc_uchar16)\n\
INSTANCE_NORM_8_TO_F16_IMPL_2D(I8_F32toF16, vxc_char16)\n\
"; /* end of instance_normalization_1_vx*/

static const char instance_normalization_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniSum_X_X2_8x2;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_scale2;\n\
_viv_uniform float sum_x_tail;\n\
_viv_uniform float sum_x2_tail0;\n\
_viv_uniform float sum_x2_tail1;\n\
\n\
#define INSTANCE_NORM_SUMS_16BITS_IMPL(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_sums_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int height) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 3; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz); \\\n\
    vxc_short8 src0; \\\n\
    src_type in_h; \\\n\
    float4 sumsqr; \\\n\
    float4 tmpSumSqr = (float4)(0); \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    if(gidx < width) \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src0, input, coord, 0, \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            _viv_asm(COPY, in_h, src0, 16); \\\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), \\\n\
                uniSum_X_X2_8x2); \\\n\
            tmpSumSqr += sumsqr; \\\n\
        } \\\n\
        tmpSumSqr.y = tmpSumSqr.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * tmpSumSqr.x; \\\n\
        tmpSumSqr.x = tmpSumSqr.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
 \\\n\
    lcl_sum[lidx] = tmpSumSqr.x; \\\n\
    lcl_sqr[lidx] = tmpSumSqr.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        float sum = 0; \\\n\
        float sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 data = (float4)(sum, sqr, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_SUMS_16BITS_IMPL(F16, vxc_half8)\n\
INSTANCE_NORM_SUMS_16BITS_IMPL(I16, vxc_short8)\n\
\n\
#define INSTANCE_NORM_SUMS_16BITS_IMPL_2D(name, src_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_sums_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float eps, int height) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 3; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int gidy = gidz * height; \\\n\
 \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    vxc_short8 src0; \\\n\
    src_type in_h; \\\n\
    float4 sumsqr; \\\n\
    float4 tmpSumSqr = (float4)(0); \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    int endH = gidy + height; \\\n\
    if(gidx < width) \\\n\
    { \\\n\
        for(; coord.y < endH;) \\\n\
        { \\\n\
            VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            _viv_asm(COPY, in_h, src0, 16); \\\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), \\\n\
                uniSum_X_X2_8x2); \\\n\
            tmpSumSqr += sumsqr; \\\n\
        } \\\n\
        tmpSumSqr.y = tmpSumSqr.y * input_scale2 + sum_x2_tail0 + sum_x2_tail1 * tmpSumSqr.x; \\\n\
        tmpSumSqr.x = tmpSumSqr.x * input_scale + sum_x_tail; \\\n\
    } \\\n\
 \\\n\
    lcl_sum[lidx] = tmpSumSqr.x; \\\n\
    lcl_sqr[lidx] = tmpSumSqr.y; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        float sum = 0; \\\n\
        float sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 data = (float4)(sum, sqr, 0, 0); \\\n\
        write_imagef(output, coord_out, data); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_SUMS_16BITS_IMPL_2D(F16, vxc_half8)\n\
INSTANCE_NORM_SUMS_16BITS_IMPL_2D(I16, vxc_short8)\n\
\n\
#define INSTANCE_NORM_16BITS_IMPL(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel void instance_norm_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       means, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int height) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_para = (int4)(0, gidz, 0, 0); \\\n\
    vxc_short8 src0; \\\n\
    src_type in_h; \\\n\
    float4 coef; \\\n\
 \\\n\
    coef = read_imagef(means, coord_para.xy); \\\n\
 \\\n\
    float4  tmpData0, tmpData1; \\\n\
    copy_type outval; \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    dst_type dst; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, in_h, src0, 16); \\\n\
 \\\n\
    coord_in.y ++; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
 \\\n\
    float4 norm; \\\n\
    norm = tmpData0 * coef.x + coef.y; \\\n\
    _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
    norm = tmpData1 * coef.x + coef.y; \\\n\
    _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_16BITS_IMPL(F16_F32toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
INSTANCE_NORM_16BITS_IMPL(F16_F32toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
INSTANCE_NORM_16BITS_IMPL(F16_F32toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
INSTANCE_NORM_16BITS_IMPL(F16_F32toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
INSTANCE_NORM_16BITS_IMPL(I16_F32toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
INSTANCE_NORM_16BITS_IMPL(I16_F32toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
\n\
#define INSTANCE_NORM_16BITS_IMPL_2D(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel void instance_norm_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       means, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int height) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int gidy = gidz * height; \\\n\
    int4 coord = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    int4 coord_para = (int4)(0, gidz, 0, 0); \\\n\
    int endH = gidy + height; \\\n\
    vxc_short8 src0; \\\n\
    src_type in_h; \\\n\
    float4 coef; \\\n\
 \\\n\
    coef = read_imagef(means, coord_para.xy); \\\n\
 \\\n\
    float4  tmpData0, tmpData1; \\\n\
    copy_type outval; \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(; coord.y < endH; coord.y++) \\\n\
    { \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, in_h, src0, 16); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_0_4x4); \\\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataToFP32_1_4x4); \\\n\
    float4 norm; \\\n\
    norm = tmpData0 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal0, norm); \\\n\
    norm = tmpData1 * coef.x + coef.y; \\\n\
    _viv_asm(CONV, tmpVal1, norm); \\\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, outval, dst, 16); \\\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCE_NORM_16BITS_IMPL_2D(F16_F32toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
INSTANCE_NORM_16BITS_IMPL_2D(F16_F32toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
INSTANCE_NORM_16BITS_IMPL_2D(F16_F32toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
INSTANCE_NORM_16BITS_IMPL_2D(F16_F32toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
INSTANCE_NORM_16BITS_IMPL_2D(I16_F32toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
INSTANCE_NORM_16BITS_IMPL_2D(I16_F32toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
"; /* end of instance_normalization_2_vx*/

static const char instance_normalization_3_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_sums_BF16(\n\
    image2d_array_t input, image2d_array_t output, float eps, int height)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz);\n\
    vxc_short8 src0, src1, src2;\n\
    float4 srcA, srcB;\n\
    float sum = 0, sqr = 0;\n\
    float4 one = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part0_2x8);\n\
            VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, srcA, src1, 16);\n\
            _viv_asm(COPY, srcB, src2, 16);\n\
            sum += dot(srcA, one) + dot(srcB, one);\n\
            sqr += dot(srcA * srcA, one) + dot(srcB * srcB, one);\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0;\n\
        sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_sums_BF16_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int height)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_short8 src0, src1, src2;\n\
    float4 srcA, srcB;\n\
    float sum = 0, sqr = 0;\n\
    float4 one = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part0_2x8);\n\
            VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, srcA, src1, 16);\n\
            _viv_asm(COPY, srcB, src2, 16);\n\
            sum += dot(srcA, one) + dot(srcB, one);\n\
            sqr += dot(srcA * srcA, one) + dot(srcB * srcB, one);\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0;\n\
        sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_BF16_F32toBF16(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       means,\n\
    __write_only image2d_array_t output,\n\
                 int height)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    vxc_short8 src0, src1, src2;\n\
    float4 coef;\n\
\n\
    coef = read_imagef(means, coord.yz);\n\
\n\
    float4  tmpData0, tmpData1;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y ++;\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, tmpData0, src1, 16);\n\
    _viv_asm(COPY, tmpData1, src2, 16);\n\
\n\
    float4 norm;\n\
    norm = tmpData0 * coef.x + coef.y;\n\
    _viv_asm(COPY, src0, norm, 16);\n\
    norm = tmpData1 * coef.x + coef.y;\n\
    _viv_asm(COPY, src1, norm, 16);\n\
    VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, src2, \\\n\
                    VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_BF16_F32toBF16_2D(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       means,\n\
    __write_only image2d_array_t output,\n\
                 int height)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
    vxc_short8 src0, src1, src2;\n\
    float4 coef;\n\
\n\
    coef = read_imagef(means, coord_para);\n\
\n\
    float4  tmpData0, tmpData1;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, tmpData0, src1, 16);\n\
    _viv_asm(COPY, tmpData1, src2, 16);\n\
\n\
    float4 norm;\n\
    norm = tmpData0 * coef.x + coef.y;\n\
    _viv_asm(COPY, src0, norm, 16);\n\
    norm = tmpData1 * coef.x + coef.y;\n\
    _viv_asm(COPY, src1, norm, 16);\n\
    VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of instance_normalization_3_vx*/

static const char l1norm_axis0_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define epsilon 1e-12\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Bin_2x8;\n\
_viv_uniform VXC_512Bits ExtractBin_part0_4x4;\n\
_viv_uniform VXC_512Bits ExtractBin_part1_4x4;\n\
\n\
\n\
\n\
#define L1_NORM_AXIS0_SH_2D(name0, name1, src_type, conv_type0, conv_type1, dst_type, save_type) \\\n\
__kernel void l1norm_##name0##to##name1##_2D_axis0 \\\n\
    ( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int2 coord = (int2)(0, get_global_id(0)); \\\n\
    src_type    v0, v1; \\\n\
    conv_type0  src0, src1; \\\n\
    conv_type1  dst0, dst1; \\\n\
    dst_type    dst; \\\n\
    save_type   out; \\\n\
    float4 src0_f, src1_f, src2_f, src3_f; \\\n\
 \\\n\
    float4 sum = 0; \\\n\
    float4 total = 0; \\\n\
    float4 rcp_total = 0; \\\n\
    half4  rcp_total_half = 0; \\\n\
    float4 one4 = (float4)(1.0f, 1.0f, 1.0f, 1.0f); \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        VXC_ReadImage(v1, input, coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src1, v1, 16); \\\n\
        coord.x = coord.x + 16; \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src2_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        VXC_DP4x4(src3_f, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = fabs(src0_f - inputZp); \\\n\
        src1_f = fabs(src1_f - inputZp); \\\n\
        src2_f = fabs(src2_f - inputZp); \\\n\
        src3_f = fabs(src3_f - inputZp); \\\n\
        sum = src0_f + src1_f + src2_f + src3_f; \\\n\
        total = total + dot(sum, one4); \\\n\
    } while (coord.x < axis_size); \\\n\
 \\\n\
    total = total > epsilon ? total : epsilon; \\\n\
    rcp_total = 1 / total * outputscale; \\\n\
    coord.x = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = (src0_f - inputZp) * rcp_total.x + outputtail; \\\n\
        src1_f = (src1_f - inputZp) * rcp_total.x + outputtail; \\\n\
        _viv_asm(CONV_RTE, dst0, src0_f); \\\n\
        _viv_asm(CONV_RTE, dst1, src1_f); \\\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Bin_2x8); \\\n\
        _viv_asm(COPY, out, dst, 16); \\\n\
        VXC_WriteImage(output, coord, out, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x = coord.x + 8; \\\n\
    } while (coord.x < axis_size); \\\n\
}\n\
L1_NORM_AXIS0_SH_2D(U8, F16,vxc_uchar8,vxc_uchar8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS0_SH_2D(U8, U8, vxc_uchar8,vxc_uchar8,short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS0_SH_2D(I8, F16,vxc_char8, vxc_char8, half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS0_SH_2D(I8, I8, vxc_char8, vxc_char8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS0_SH_2D(I16,F16,vxc_short8,vxc_short8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS0_SH_2D(I16,I16,vxc_short8,vxc_short8,short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS0_SH_2D(F16,U8, vxc_short8,vxc_half8, short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS0_SH_2D(F16,I8, vxc_short8,vxc_half8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS0_SH_2D(F16,I16,vxc_short8,vxc_half8, short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS0_SH_2D(F16,F16,vxc_short8,vxc_half8, half4, vxc_half8, vxc_short8)\n\
\n\
\n\
#define L1_NORM_AXIS0_SH(name0, name1, src_type, conv_type0, conv_type1, dst_type, save_type) \\\n\
__kernel void l1norm_##name0##to##name1##_axis0 \\\n\
    ( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type    v0, v1; \\\n\
    conv_type0  src0, src1; \\\n\
    conv_type1  dst0, dst1; \\\n\
    dst_type    dst; \\\n\
    save_type   out; \\\n\
    float4 src0_f, src1_f, src2_f, src3_f; \\\n\
 \\\n\
    float4 sum = 0; \\\n\
    float4 total = 0; \\\n\
    float4 rcp_total = 0; \\\n\
    half4  rcp_total_half = 0; \\\n\
    float4 one4 = (float4)(1.0f, 1.0f, 1.0f, 1.0f); \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        VXC_ReadImage2DArray(v1, input, coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src1, v1, 16); \\\n\
        coord.x = coord.x + 16; \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src2_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        VXC_DP4x4(src3_f, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = fabs(src0_f - inputZp); \\\n\
        src1_f = fabs(src1_f - inputZp); \\\n\
        src2_f = fabs(src2_f - inputZp); \\\n\
        src3_f = fabs(src3_f - inputZp); \\\n\
        sum = src0_f + src1_f + src2_f + src3_f; \\\n\
        total = total + dot(sum, one4); \\\n\
    } while (coord.x < axis_size); \\\n\
 \\\n\
    total = total > epsilon ? total : epsilon; \\\n\
    rcp_total = 1 / total * outputscale; \\\n\
    coord.x = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = (src0_f - inputZp) * rcp_total.x + outputtail; \\\n\
        src1_f = (src1_f - inputZp) * rcp_total.x + outputtail; \\\n\
        _viv_asm(CONV_RTE, dst0, src0_f); \\\n\
        _viv_asm(CONV_RTE, dst1, src1_f); \\\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Bin_2x8); \\\n\
        _viv_asm(COPY, out, dst, 16); \\\n\
        VXC_WriteImage2DArray(output, coord, out, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x = coord.x + 8; \\\n\
    } while (coord.x < axis_size); \\\n\
}\n\
L1_NORM_AXIS0_SH(U8, F16,vxc_uchar8,vxc_uchar8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS0_SH(U8, U8, vxc_uchar8,vxc_uchar8,short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS0_SH(I8, F16,vxc_char8, vxc_char8, half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS0_SH(I8, I8, vxc_char8, vxc_char8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS0_SH(I16,F16,vxc_short8,vxc_short8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS0_SH(I16,I16,vxc_short8,vxc_short8,short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS0_SH(F16,U8, vxc_short8,vxc_half8, short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS0_SH(F16,I8, vxc_short8,vxc_half8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS0_SH(F16,I16,vxc_short8,vxc_half8, short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS0_SH(F16,F16,vxc_short8,vxc_half8, half4, vxc_half8, vxc_short8)\n\
\n\
\n\
"; /* end of l1norm_axis0_vx*/

static const char l1norm_axis1_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define epsilon 1e-12\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Bin_2x8;\n\
_viv_uniform VXC_512Bits ExtractBin_part0_4x4;\n\
_viv_uniform VXC_512Bits ExtractBin_part1_4x4;\n\
\n\
\n\
#define L1_NORM_AXIS1_SH_2D(name0, name1, src_type, conv_type0, conv_type1, dst_type, save_type) \\\n\
__kernel void l1norm_##name0##to##name1##_2D_axis1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0),0); \\\n\
    src_type    v0; \\\n\
    conv_type0  src0; \\\n\
    dst_type    dst; \\\n\
    conv_type1  dst0, dst1; \\\n\
    save_type   out; \\\n\
    float4 src0_f, src1_f; \\\n\
 \\\n\
    float4 total0 = 0, total1 = 0; \\\n\
    float4 rcp_total0 = 0, rcp_total1 = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        coord.y = coord.y + 1; \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = fabs(src0_f - inputZp); \\\n\
        src1_f = fabs(src1_f - inputZp); \\\n\
        total0 = total0 + src0_f; \\\n\
        total1 = total1 + src1_f; \\\n\
    } while (coord.y < axis_size); \\\n\
 \\\n\
    total0 = total0 > epsilon ? total0 : epsilon; \\\n\
    total1 = total1 > epsilon ? total1 : epsilon; \\\n\
    rcp_total0 = 1 / total0 * outputscale; \\\n\
    rcp_total1 = 1 / total1 * outputscale; \\\n\
    coord.y = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = (src0_f - inputZp) * rcp_total0 + outputtail; \\\n\
        src1_f = (src1_f - inputZp) * rcp_total1 + outputtail; \\\n\
        _viv_asm(CONV_RTE, dst0, src0_f); \\\n\
        _viv_asm(CONV_RTE, dst1, src1_f); \\\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Bin_2x8); \\\n\
        _viv_asm(COPY, out, dst, 16); \\\n\
        VXC_WriteImage(output, coord, out, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y = coord.y + 1; \\\n\
    } while (coord.y < axis_size); \\\n\
}\n\
L1_NORM_AXIS1_SH_2D(U8, F16,vxc_uchar8,vxc_uchar8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS1_SH_2D(U8, U8, vxc_uchar8,vxc_uchar8,short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS1_SH_2D(I8, F16,vxc_char8, vxc_char8, half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS1_SH_2D(I8, I8, vxc_char8, vxc_char8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS1_SH_2D(I16,F16,vxc_short8,vxc_short8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS1_SH_2D(I16,I16,vxc_short8,vxc_short8,short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS1_SH_2D(F16,U8, vxc_short8,vxc_half8, short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS1_SH_2D(F16,I8, vxc_short8,vxc_half8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS1_SH_2D(F16,I16,vxc_short8,vxc_half8, short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS1_SH_2D(F16,F16,vxc_short8,vxc_half8, half4, vxc_half8, vxc_short8)\n\
\n\
\n\
#define L1_NORM_AXIS1_SH(name0, name1, src_type, conv_type0, conv_type1, dst_type, save_type) \\\n\
__kernel void l1norm_##name0##to##name1##_axis1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    src_type    v0; \\\n\
    conv_type0  src0; \\\n\
    dst_type    dst; \\\n\
    conv_type1  dst0, dst1; \\\n\
    save_type   out; \\\n\
    float4 src0_f, src1_f; \\\n\
 \\\n\
    float4 total0 = 0, total1 = 0; \\\n\
    float4 rcp_total0 = 0, rcp_total1 = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        coord.y = coord.y + 1; \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = fabs(src0_f - inputZp); \\\n\
        src1_f = fabs(src1_f - inputZp); \\\n\
        total0 = total0 + src0_f; \\\n\
        total1 = total1 + src1_f; \\\n\
    } while (coord.y < axis_size); \\\n\
 \\\n\
    total0 = total0 > epsilon ? total0 : epsilon; \\\n\
    total1 = total1 > epsilon ? total1 : epsilon; \\\n\
    rcp_total0 = 1 / total0 * outputscale; \\\n\
    rcp_total1 = 1 / total1 * outputscale; \\\n\
    coord.y = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = (src0_f - inputZp) * rcp_total0 + outputtail; \\\n\
        src1_f = (src1_f - inputZp) * rcp_total1 + outputtail; \\\n\
        _viv_asm(CONV_RTE, dst0, src0_f); \\\n\
        _viv_asm(CONV_RTE, dst1, src1_f); \\\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Bin_2x8); \\\n\
        _viv_asm(COPY, out, dst, 16); \\\n\
        VXC_WriteImage2DArray(output, coord, out, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y = coord.y + 1; \\\n\
    } while (coord.y < axis_size); \\\n\
}\n\
L1_NORM_AXIS1_SH(U8, F16,vxc_uchar8,vxc_uchar8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS1_SH(U8, U8, vxc_uchar8,vxc_uchar8,short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS1_SH(I8, F16,vxc_char8, vxc_char8, half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS1_SH(I8, I8, vxc_char8, vxc_char8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS1_SH(I16,F16,vxc_short8,vxc_short8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS1_SH(I16,I16,vxc_short8,vxc_short8,short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS1_SH(F16,U8, vxc_short8,vxc_half8, short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS1_SH(F16,I8, vxc_short8,vxc_half8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS1_SH(F16,I16,vxc_short8,vxc_half8, short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS1_SH(F16,F16,vxc_short8,vxc_half8, half4, vxc_half8, vxc_short8)\n\
"; /* end of l1norm_axis1_vx*/

static const char l1norm_axis2_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define epsilon 1e-12\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Bin_2x8;\n\
_viv_uniform VXC_512Bits ExtractBin_part0_4x4;\n\
_viv_uniform VXC_512Bits ExtractBin_part1_4x4;\n\
\n\
\n\
#define L1_NORM_AXIS2_SH(name0, name1, src_type, conv_type0, conv_type1, dst_type, save_type) \\\n\
__kernel void l1norm_##name0##to##name1##_axis2 \\\n\
    ( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type    v0; \\\n\
    conv_type0  src0; \\\n\
    dst_type    dst; \\\n\
    conv_type1  dst0, dst1; \\\n\
    save_type   out; \\\n\
    float4 src0_f, src1_f; \\\n\
 \\\n\
    float4 total0 = 0, total1 = 0; \\\n\
    float4 rcp_total0 = 0, rcp_total1 = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        coord.z = coord.z + 1; \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = fabs(src0_f - inputZp); \\\n\
        src1_f = fabs(src1_f - inputZp); \\\n\
        total0 = total0 + src0_f; \\\n\
        total1 = total1 + src1_f; \\\n\
    } while (coord.z < axis_size); \\\n\
 \\\n\
    total0 = total0 > epsilon ? total0 : epsilon; \\\n\
    total1 = total1 > epsilon ? total1 : epsilon; \\\n\
    rcp_total0 = 1 / total0 * outputscale; \\\n\
    rcp_total1 = 1 / total1 * outputscale; \\\n\
    coord.z = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(v0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, v0, 16); \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part0_4x4); \\\n\
        VXC_DP4x4(src1_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), ExtractBin_part1_4x4); \\\n\
        src0_f = (src0_f - inputZp) * rcp_total0 + outputtail; \\\n\
        src1_f = (src1_f - inputZp) * rcp_total1 + outputtail; \\\n\
        _viv_asm(CONV_RTE, dst0, src0_f); \\\n\
        _viv_asm(CONV_RTE, dst1, src1_f); \\\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Bin_2x8); \\\n\
        _viv_asm(COPY, out, dst, 16); \\\n\
        VXC_WriteImage2DArray(output, coord, out, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z = coord.z + 1; \\\n\
    } while (coord.z < axis_size); \\\n\
}\n\
L1_NORM_AXIS2_SH(U8, F16,vxc_uchar8,vxc_uchar8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS2_SH(U8, U8, vxc_uchar8,vxc_uchar8,short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS2_SH(I8, F16,vxc_char8, vxc_char8, half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS2_SH(I8, I8, vxc_char8, vxc_char8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS2_SH(I16,F16,vxc_short8,vxc_short8,half4, vxc_half8, vxc_short8)\n\
L1_NORM_AXIS2_SH(I16,I16,vxc_short8,vxc_short8,short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS2_SH(F16,U8, vxc_short8,vxc_half8, short4,vxc_uchar8,vxc_uchar8)\n\
L1_NORM_AXIS2_SH(F16,I8, vxc_short8,vxc_half8, short4,vxc_char8, vxc_char8)\n\
L1_NORM_AXIS2_SH(F16,I16,vxc_short8,vxc_half8, short4,vxc_short8,vxc_short8)\n\
L1_NORM_AXIS2_SH(F16,F16,vxc_short8,vxc_half8, half4, vxc_half8, vxc_short8)\n\
"; /* end of l1norm_axis2_vx*/

static const char l2normalizescale_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define VXC_Vstore3(Pointer, Offset, Data)   \\\n\
do \\\n\
{ int byteOffset = ((int)sizeof((Data)))*(Offset); \\\n\
VXC_OP3_NoDest(vstore3, Pointer, byteOffset, Data); } \\\n\
while(0)\n\
\n\
#define L2NORMSCALE_SWITCH_PROCESS(case_value, vec_val, ZpValue) \\\n\
                switch (case_value) \\\n\
                { \\\n\
                    case 1: \\\n\
                        vec_val.s123  = ZpValue; \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 2: \\\n\
                        vec_val.s23  = ZpValue; \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 3: \\\n\
                        vec_val.s3  = ZpValue; \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 4: \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 5: \\\n\
                        vec_val.s567 = ZpValue; \\\n\
                    break; \\\n\
                    case 6: \\\n\
                        vec_val.s67 = ZpValue; \\\n\
                    break; \\\n\
                    case 7: \\\n\
                        vec_val.s7 = ZpValue; \\\n\
                    break; \\\n\
                    default: \\\n\
                    break; \\\n\
                }\n\
\n\
#define L2NORMSCALE_REM_PROCESS(ZpValue) \\\n\
            VXC_Vload8(src0, src_ptr, 0); \\\n\
            VXC_Vload8(src1, src_ptr, 1); \\\n\
            if (inputRemain <= 8) \\\n\
            { \\\n\
                L2NORMSCALE_SWITCH_PROCESS(inputRemain, src0, ZpValue) \\\n\
                src1 = 0; \\\n\
            } \\\n\
            else if (inputRemain < 16) \\\n\
            { \\\n\
                int inputRemain8 = inputRemain - 8; \\\n\
                L2NORMSCALE_SWITCH_PROCESS(inputRemain8, src1, ZpValue) \\\n\
            }\n\
\n\
\n\
#define L2NORMSCALE_MUL_PROCESS(index) \\\n\
        VXC_Vload8(src0, src_ptr, index); \\\n\
        _viv_asm(COPY, val0, src0, 16); \\\n\
        VXC_Vload8(scale_s16, scale_ptr, index); \\\n\
        _viv_asm(COPY, scale_f16, scale_s16, 16); \\\n\
        _viv_asm(COPY, input_ZP, inputZP, 4); \\\n\
        VXC_DP4x4(vec0, val0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part0_4x4);\\\n\
        VXC_DP4x4(vec1, val0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part1_4x4);\\\n\
        VXC_DP4x4(scale_f32, scale_f16, scale_f16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniFp16toFp32_4x4);\\\n\
        VXC_DP4x4(scale1_f32, scale_f16, scale_f16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniFp16toFp32Hi_4x4);\\\n\
        vec0 = vec0 * rsqrt0.xxxx + output_ZP;\\\n\
        vec1 = vec1 * rsqrt0.xxxx + output_ZP;\\\n\
        vec0 *= scale_f32;\\\n\
        vec1 *= scale1_f32;\\\n\
        _viv_asm(CONV_RTE, dst0, vec0);\\\n\
        _viv_asm(CONV_RTE, dst1, vec1);\\\n\
        VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
        _viv_asm(COPY, dst, dst2, 16);\n\
\n\
_viv_uniform int inputWidth;\n\
_viv_uniform int inputWidthRemain256;\n\
_viv_uniform int inputWidthCount;\n\
_viv_uniform VXC_512Bits uniSumSqrt_16x1;\n\
_viv_uniform float r_inputScale;\n\
\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bin_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32Hi_4x4;\n\
_viv_uniform float IntergerScale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform int inputWidthRemain128;\n\
_viv_uniform float zP2x;\n\
_viv_uniform float zpSqrt16x;\n\
_viv_uniform VXC_512Bits uniSumAll_16x1;\n\
_viv_uniform int inputZP;\n\
\n\
\n\
#define L2NORMSCALE_MUL_AXIS0_PROCESS(dst_type, convert_type, output_type, copy_type) \\\n\
    vxc_float4 rsqrt0;\\\n\
    Image dst_img = create_image_from_image2d(output, 1); \\\n\
    dst_type  *dst_ptr = (dst_type *)dst_img.ptr; \\\n\
    Image s_img = create_image_from_image2d(scale, 2); \\\n\
    short *scale_ptr = (short *)s_img.ptr; \\\n\
    vxc_float4 vec0, vec1;\\\n\
    convert_type dst0, dst1;\\\n\
    vxc_short8 scale_s16;\\\n\
    vxc_half8  scale_f16;\\\n\
    vxc_float4 scale_f32, scale1_f32;\\\n\
    output_type dst2;\\\n\
    copy_type dst;\\\n\
    rsqrt0 = sum.xxxx * IntergerScale;\\\n\
    src_ptr = src_ptr_base + (get_global_id(0) + get_global_id(1) * inputWidth); \\\n\
    dst_ptr   += (get_global_id(0) + get_global_id(1) * inputWidth);\\\n\
    scale_ptr += get_global_id(0);\\\n\
    for(int i = 0; i < inputWidthCount; i++)\\\n\
    {\\\n\
        L2NORMSCALE_MUL_PROCESS(0) \\\n\
        VXC_Vstore8(dst_ptr, 0, dst); \\\n\
        L2NORMSCALE_MUL_PROCESS(1) \\\n\
        VXC_Vstore8(dst_ptr, 1, dst); \\\n\
        src_ptr   += 256; \\\n\
        dst_ptr   += 256; \\\n\
        scale_ptr += 256; \\\n\
    }\\\n\
    if (inputWidthRemain256) \\\n\
    { \\\n\
        offset  = get_global_id(0) + inputWidthCount * 128; \\\n\
        inputRemain = inputWidth - offset; \\\n\
        if (inputRemain >= 8) \\\n\
        { \\\n\
            L2NORMSCALE_MUL_PROCESS(0) \\\n\
            VXC_Vstore8(dst_ptr, 0, dst); \\\n\
            src_ptr   += 8; \\\n\
            dst_ptr   += 8; \\\n\
            scale_ptr += 8; \\\n\
            inputRemain -= 8; \\\n\
        } \\\n\
        if (inputRemain > 0) \\\n\
        { \\\n\
            L2NORMSCALE_MUL_PROCESS(0) \\\n\
            switch (inputRemain) \\\n\
            { \\\n\
                case 1: \\\n\
                    dst_ptr[0] = dst.s0; \\\n\
                break; \\\n\
                case 2: \\\n\
                    VXC_Vstore2(dst_ptr, 0, dst.s01); \\\n\
                break; \\\n\
                case 3: \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst.s012); \\\n\
                break; \\\n\
                case 4: \\\n\
                    VXC_Vstore4(dst_ptr, 0, dst.s0123); \\\n\
                break; \\\n\
                case 5: \\\n\
                    VXC_Vstore2(dst_ptr, 0, dst.s01); \\\n\
                    dst.s012 = dst.s234; \\\n\
                    dst_ptr += 2; \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst.s012); \\\n\
                break; \\\n\
                case 6: \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst.s012); \\\n\
                    dst.s012 = dst.s345; \\\n\
                    dst_ptr += 3; \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst.s012); \\\n\
                break; \\\n\
                case 7: \\\n\
                    VXC_Vstore4(dst_ptr, 0, dst.s0123); \\\n\
                     dst.s012 = dst.s456; \\\n\
                    dst_ptr += 4; \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst.s012); \\\n\
                break; \\\n\
                default: \\\n\
                    VXC_Vstore8(dst_ptr, 0, dst); \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
\n\
\n\
#define L2NORMSCALE_AXIS0(in0_name, in1_name, out_name, read_type, read_type2, src_type, INPUTSCALE, \\\n\
                            dst_type, convert_type, output_type, copy_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) \\\n\
     void l2normalizescale_axis0_##in0_name##_##in1_name##to##out_name \\\n\
    (\\\n\
    __read_only  image2d_t input, __read_only  image2d_t scale, __write_only image2d_t output,\\\n\
    int axis )\\\n\
{ \\\n\
    int lidx = get_local_id(0); \\\n\
    int offset  = get_global_id(0); \\\n\
    Image src_img = create_image_from_image2d(input, 1); \\\n\
    read_type *src_ptr_base = (read_type *)src_img.ptr; \\\n\
    read_type *src_ptr; \\\n\
    read_type2 src0, src1; \\\n\
    src_type   val0, val1; \\\n\
    int   inputRemain; \\\n\
    vxc_float4 sum = {0.0f}; \\\n\
    read_type2 input_ZP ;\\\n\
    __local float lcl_sum[16]; \\\n\
    src_ptr = src_ptr_base + (get_global_id(0) + get_global_id(1) * inputWidth); \\\n\
    for (int i = 0; i < inputWidthCount; i++) \\\n\
    { \\\n\
        VXC_Vload8(src0, src_ptr, 0); \\\n\
        VXC_Vload8(src1, src_ptr, 1); \\\n\
        _viv_asm(COPY, val0, src0, 16); \\\n\
        _viv_asm(COPY, val1, src1, 16); \\\n\
        VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
            uniSumSqrt_16x1); \\\n\
        sum.x += sum.y;  \\\n\
        src_ptr += 256; \\\n\
    } \\\n\
    if (inputWidthRemain256) \\\n\
    { \\\n\
        offset  = get_global_id(0) + inputWidthCount * 256;\\\n\
        inputRemain = inputWidth - offset; \\\n\
        if (inputRemain > 0) \\\n\
        { \\\n\
            L2NORMSCALE_REM_PROCESS(0) \\\n\
            _viv_asm(COPY, val0, src0, 16); \\\n\
            _viv_asm(COPY, val1, src1, 16); \\\n\
            VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
                uniSumSqrt_16x1); \\\n\
            sum.x += sum.y; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum.x; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    sum.x = dot(data0, one); \\\n\
    sum.x = rsqrt(sum.x) * INPUTSCALE; \\\n\
    L2NORMSCALE_MUL_AXIS0_PROCESS(dst_type, convert_type, output_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS0(F16, F16, F16, ushort, vxc_ushort8, vxc_half8, 1, \\\n\
                     ushort, half4, vxc_half8, vxc_ushort8)\n\
\n\
#define L2NORMSCALE_AXIS0_QNT(in0_name, in1_name, out_name,\\\n\
                    src_type, src_scalar_type, dst_type, convert_type, output_type, copy_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) \\\n\
void l2normalizescale_axis0_##in0_name##_##in1_name##to##out_name \\\n\
    (\\\n\
    __read_only image2d_t input, __read_only image2d_t scale, __write_only image2d_t output, int axis)\\\n\
{ \\\n\
    int lidx = get_local_id(0); \\\n\
    int offset  = get_global_id(0); \\\n\
    Image src_img = create_image_from_image2d(input, 1); \\\n\
    uchar *src_ptr_base = (uchar *)src_img.ptr; \\\n\
    uchar *src_ptr; \\\n\
    src_type src0, src1; \\\n\
    src_type val0, val1; \\\n\
    int   inputRemain; \\\n\
    vxc_float4 sum = {0.0f}; \\\n\
    vxc_uchar8 input_ZP ; \\\n\
    __local float lcl_sum[16]; \\\n\
    src_ptr = src_ptr_base + (get_global_id(0) + get_global_id(1) * inputWidth); \\\n\
    for (int i = 0; i < inputWidthCount; i++) \\\n\
    { \\\n\
        VXC_Vload8(val0, src_ptr, 0); \\\n\
        VXC_Vload8(val1, src_ptr, 1); \\\n\
        VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
            uniSumSqrt_16x1); \\\n\
        VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 1),\\\n\
            uniSumAll_16x1); \\\n\
        sum.w  = sum.y - zP2x * sum.z + zpSqrt16x; \\\n\
        sum.x += sum.w; \\\n\
        src_ptr += 256; \\\n\
    } \\\n\
    if (inputWidthRemain256) \\\n\
    { \\\n\
        offset  = get_global_id(0) + inputWidthCount * 256; \\\n\
        inputRemain = inputWidth - offset; \\\n\
        if (inputRemain > 0) \\\n\
        { \\\n\
            L2NORMSCALE_REM_PROCESS((src_scalar_type)inputZP) \\\n\
            _viv_asm(COPY, val0, src0, 16); \\\n\
            _viv_asm(COPY, val1, src1, 16); \\\n\
            VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
                uniSumSqrt_16x1); \\\n\
            VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 1),\\\n\
                uniSumAll_16x1); \\\n\
            sum.w  = sum.y - zP2x * sum.z + zpSqrt16x; \\\n\
            sum.x += sum.w; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum.x; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    sum.x = dot(data0, one); \\\n\
    sum.x = rsqrt(sum.x) * r_inputScale; \\\n\
    L2NORMSCALE_MUL_AXIS0_PROCESS(dst_type, convert_type, output_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS0_QNT(U8,  F16, F16, vxc_uchar8, uchar, ushort, half4, vxc_half8,  vxc_ushort8)\n\
L2NORMSCALE_AXIS0_QNT(U8,  F16, U8,  vxc_uchar8, uchar, uchar,  int4,  vxc_uchar8, vxc_uchar8)\n\
L2NORMSCALE_AXIS0_QNT(I8,  F16, F16, vxc_char8,  char,  ushort, half4, vxc_half8,  vxc_ushort8)\n\
L2NORMSCALE_AXIS0_QNT(I8,  F16, I8,  vxc_char8,  char,  char,   int4,  vxc_char8,  vxc_char8)\n\
L2NORMSCALE_AXIS0_QNT(I16, F16, F16, vxc_short8, short, ushort, half4, vxc_half8,  vxc_ushort8)\n\
L2NORMSCALE_AXIS0_QNT(I16, F16, I16, vxc_short8, short, short,  int4,  vxc_short8, vxc_short8)\n\
"; /* end of l2normalizescale_axis0_vx*/

static const char l2normalizescale_axis0_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int inputWidth;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float zP2x;\n\
_viv_uniform int inputZP;\n\
\n\
_viv_uniform float inOutScale;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float zpSqr8x;\n\
_viv_uniform VXC_512Bits uniInt16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1)))\n\
     void l2normalizescale_axis0_F16_F16toF16_2D(\n\
    __read_only image2d_t input, __read_only image2d_t scale, __write_only image2d_t output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int lidx = get_local_id(0);\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 in_h, scale_h, tmpDst;\n\
    float sum = 0;\n\
    vxc_float4 scale_f0, scale_f1, sumsqr, tmpData0, tmpData1;\n\
    __local float lcl_sum[16];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    for(; coord.x < inputWidth;)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.x += 128;\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniFp16SumSqr_dp8x2);\n\
        sum += sumsqr.y;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    float alpha = rsqrt(sum);\n\
\n\
    for(coord.x = get_global_id(0); coord.x < inputWidth; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, scale, coord.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
\n\
        half4 tmpVal0, tmpVal1;\n\
        tmpData0 *= scale_f0 * alpha;\n\
        tmpData1 *= scale_f1 * alpha;\n\
        _viv_asm(CONV, tmpVal0, tmpData0);\n\
        _viv_asm(CONV, tmpVal1, tmpData1);\n\
        VXC_DP2x8(tmpDst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertHalfToFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpDst, 16);\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
\n\
#define L2NORMSCALE_QINTF16TOQINT_AXIS0_2D(in0_name, in1_name, out_name, read_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) \\\n\
     void l2normalizescale_axis0_##in0_name##_##in1_name##to##out_name##_2D( \\\n\
    __read_only image2d_t input, __read_only image2d_t scale, __write_only image2d_t output, int axis)\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    int lidx = get_local_id(0); \\\n\
    read_type src0, dst; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0; \\\n\
    vxc_float4 scale_f0, scale_f1, sumsqr; \\\n\
    __local float lcl_sum[16]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    for(; coord.x < inputWidth;) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 128; \\\n\
        VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniInt16SumSqr_dp8x2); \\\n\
        sum += sumsqr.y - zP2x * sumsqr.x + zpSqr8x; \\\n\
    } \\\n\
    sum *= e2InScale; \\\n\
    lcl_sum[lidx] = sum; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    sum = dot(data0, one); \\\n\
    float alpha = rsqrt(sum) * inOutScale; \\\n\
    short zp = inputZP; \\\n\
    vxc_float4  tmpData0, tmpData1; \\\n\
    for(coord.x = get_global_id(0); coord.x < inputWidth; coord.x += 128) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1, scale, coord.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4); \\\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4); \\\n\
 \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4); \\\n\
 \\\n\
        int4 tmpVal0 = convert_int4_rte(tmpData0 * scale_f0 * alpha + output_ZP); \\\n\
        int4 tmpVal1 = convert_int4_rte(tmpData1 * scale_f1 * alpha + output_ZP); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
\n\
L2NORMSCALE_QINTF16TOQINT_AXIS0_2D(U8,  F16,  U8,   vxc_uchar8)\n\
L2NORMSCALE_QINTF16TOQINT_AXIS0_2D(I8,  F16,  I8,   vxc_char8)\n\
L2NORMSCALE_QINTF16TOQINT_AXIS0_2D(I16, F16,  I16,  vxc_short8)\n\
\n\
#define L2NORMSCALE_QINTF16TOF16_AXIS0_2D(in0_name, in1_name, out_name, read_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) \\\n\
     void l2normalizescale_axis0_##in0_name##_##in1_name##to##out_name##_2D( \\\n\
    __read_only image2d_t input, __read_only image2d_t scale, __write_only image2d_t output, int axis)\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    int lidx = get_local_id(0); \\\n\
    read_type src0; \\\n\
    vxc_short8 src1, dst; \\\n\
    vxc_half8 scale_h, tmpDst; \\\n\
    float sum = 0; \\\n\
    vxc_float4 scale_f0, scale_f1, sumsqr, tmpData0, tmpData1; \\\n\
    __local float lcl_sum[16]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    for(; coord.x < inputWidth;) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 128; \\\n\
        VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniInt16SumSqr_dp8x2); \\\n\
        sum += sumsqr.y - zP2x * sumsqr.x + zpSqr8x; \\\n\
    } \\\n\
    sum *= e2InScale; \\\n\
    lcl_sum[lidx] = sum; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    sum = dot(data0, one); \\\n\
    float alpha = rsqrt(sum) * inOutScale; \\\n\
    short zp = inputZP; \\\n\
    for(coord.x = get_global_id(0); coord.x < inputWidth; coord.x += 128) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1, scale, coord.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4); \\\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4); \\\n\
 \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4); \\\n\
 \\\n\
        half4 tmpVal0, tmpVal1; \\\n\
        tmpData0 *= scale_f0 * alpha; \\\n\
        tmpData1 *= scale_f1 * alpha; \\\n\
        _viv_asm(CONV, tmpVal0, tmpData0); \\\n\
        _viv_asm(CONV, tmpVal1, tmpData1); \\\n\
        VXC_DP2x8(tmpDst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertHalfToFp16_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
\n\
L2NORMSCALE_QINTF16TOF16_AXIS0_2D(U8,  F16,  F16,   vxc_uchar8)\n\
L2NORMSCALE_QINTF16TOF16_AXIS0_2D(I8,  F16,  F16,   vxc_char8)\n\
L2NORMSCALE_QINTF16TOF16_AXIS0_2D(I16, F16,  F16,   vxc_short8)\n\
"; /* end of l2normalizescale_axis0_2d_vx*/

static const char l2normalizescale_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************************************L2NormalizeScale*****************************************/\n\
_viv_uniform int L2NorS_depth;\n\
_viv_uniform VXC_512Bits UniFp16MulLo_dp4x4;\n\
_viv_uniform VXC_512Bits UniFp16MulHi_dp4x4;\n\
\n\
//int8 version\n\
_viv_uniform float r_inputScale;\n\
\n\
_viv_uniform VXC_512Bits uniUInt8SquareLo_4x4;\n\
_viv_uniform VXC_512Bits uniUInt8SquareHi_4x4;\n\
_viv_uniform int inputZP;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bin_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float IntergerScale;\n\
_viv_uniform float output_ZP;\n\
\n\
#define L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
    coord.y = get_global_id(1); \\\n\
    input_type  vect0, vect1;\\\n\
    incopy_type src0, src1;\\\n\
    vxc_float4 rsqrt0, rsqrt1;\\\n\
    rsqrt0 = sum_lo;\\\n\
    rsqrt1 = sum_hi;\\\n\
    rsqrt0 *= IntergerScale;\\\n\
    rsqrt1 *= IntergerScale;\\\n\
    for(int i = 0; i < L2NorS_depth; i += 2)\\\n\
   {\\\n\
        vxc_float4 vec0, vec1;\\\n\
        input_type input_ZP ;\\\n\
        convert_type dst0, dst1;\\\n\
        VXC_ReadImage(vect0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        _viv_asm(COPY, src0, vect0, 16); \\\n\
        VXC_ReadImage(vect1, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        _viv_asm(COPY, src1, vect1, 16); \\\n\
        vxc_short8 scale_s16;\\\n\
        vxc_half8  scale_f16;\\\n\
        vxc_float4 scale_f32;\\\n\
        VXC_ReadImage(scale_s16, scale, coord.yw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\\\n\
        _viv_asm(COPY, scale_f16, scale_s16, 16); \\\n\
        _viv_asm(COPY, input_ZP, inputZP, 4); \\\n\
        VXC_DP4x4(vec0, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part0_4x4);\\\n\
        VXC_DP4x4(vec1, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part1_4x4);\\\n\
        VXC_DP4x4(scale_f32, scale_f16, scale_f16, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardInf, 0),\\\n\
            uniFp16toFp32_4x4);\\\n\
        vec0 = vec0 * rsqrt0;\\\n\
        vec1 = vec1 * rsqrt1;\\\n\
        vec0 = vec0 * scale_f32.xxxx + output_ZP;\\\n\
        vec1 = vec1 * scale_f32.xxxx + output_ZP;\\\n\
        _viv_asm(CONV_RTE, dst0, vec0);\\\n\
        _viv_asm(CONV_RTE, dst1, vec1);\\\n\
        output_type dst2;\\\n\
        VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
        copy_type dst;\\\n\
        _viv_asm(COPY, dst, dst2, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        VXC_DP4x4(vec0, src1, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part0_4x4);\\\n\
        VXC_DP4x4(vec1, src1, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part1_4x4);\\\n\
        vec0 = vec0 * rsqrt0;\\\n\
        vec1 = vec1 * rsqrt1;\\\n\
        vec0 = vec0 * scale_f32.yyyy + output_ZP;\\\n\
        vec1 = vec1 * scale_f32.yyyy + output_ZP;\\\n\
        _viv_asm(CONV_RTE, dst0, vec0);\\\n\
        _viv_asm(CONV_RTE, dst1, vec1);\\\n\
        VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
        coord.y++;\\\n\
        _viv_asm(COPY, dst, dst2, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        coord.y++;\\\n\
    }\\\n\
\n\
\n\
#define L2NORMSCALE_AXIS1_F16_2D(in1_name, out_name,\\\n\
       input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
__kernel void l2normalizescale_axis1_F16_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_array_t input,\\\n\
    __read_only  image2d_array_t scale,\\\n\
    __write_only image2d_array_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 img1_s16, img2_s16; \\\n\
    vxc_float4 squr, sum_lo = 0, sum_hi = 0; \\\n\
    vxc_half8 img1_fp16, img2_fp16; \\\n\
    for(int i = 0; i < L2NorS_depth; i += 2) \\\n\
    { \\\n\
        VXC_ReadImage(img1_s16, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(img2_s16, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y += 2; \\\n\
        _viv_asm(COPY, img1_fp16, img1_s16, 16); \\\n\
        _viv_asm(COPY, img2_fp16, img2_s16, 16); \\\n\
        VXC_DP4x4(squr, img1_fp16, img1_fp16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulLo_dp4x4); \\\n\
        sum_lo += squr; \\\n\
        VXC_DP4x4(squr, img2_fp16, img2_fp16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulLo_dp4x4); \\\n\
        sum_lo += squr; \\\n\
        VXC_DP4x4(squr, img1_fp16, img1_fp16, VXC_MODIFIER(0, 3, 4, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulHi_dp4x4); \\\n\
        sum_hi += squr; \\\n\
        VXC_DP4x4(squr, img2_fp16, img2_fp16, VXC_MODIFIER(0, 3, 4, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulHi_dp4x4); \\\n\
        sum_hi += squr; \\\n\
    } \\\n\
    sum_lo = rsqrt(sum_lo); \\\n\
    sum_hi = rsqrt(sum_hi); \\\n\
    L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS1_F16_2D(F16, F16, vxc_short8,  vxc_half8,   vxc_half8,   half4,        vxc_short8)\n\
\n\
#define L2NORMSCALE_AXIS1_QNT_2D(in0_name, in1_name, out_name,\\\n\
       input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
__kernel void l2normalizescale_axis1_##in0_name##_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_array_t input,\\\n\
    __read_only  image2d_array_t scale,\\\n\
    __write_only image2d_array_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    input_type src0_U8, src1_U8; \\\n\
    vxc_float4 squr, sum_lo = 0, sum_hi = 0; \\\n\
    for(int i = 0; i < L2NorS_depth; i += 2) \\\n\
    { \\\n\
        vxc_short2 zero; \\\n\
        VXC_ReadImage(src0_U8, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1_U8, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y += 2; \\\n\
        _viv_asm(COPY, zero, inputZP, 4); \\\n\
        VXC_DP4x4(squr, src0_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareLo_4x4); \\\n\
        sum_lo = squr + sum_lo; \\\n\
        VXC_DP4x4(squr, src0_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareHi_4x4); \\\n\
        sum_hi = squr + sum_hi; \\\n\
        VXC_DP4x4(squr, src1_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareLo_4x4); \\\n\
        sum_lo = squr + sum_lo; \\\n\
        VXC_DP4x4(squr, src1_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareHi_4x4); \\\n\
        sum_hi = squr + sum_hi; \\\n\
    } \\\n\
    sum_lo = rsqrt(sum_lo) * r_inputScale; \\\n\
    sum_hi = rsqrt(sum_hi) * r_inputScale; \\\n\
    L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS1_QNT_2D(U8,  F16, F16, vxc_uchar16, vxc_uchar16, vxc_half8,   half4,  vxc_short8)\n\
L2NORMSCALE_AXIS1_QNT_2D(U8,  F16, U8,  vxc_uchar16, vxc_uchar16, vxc_uchar16, int4,   vxc_uchar16)\n\
L2NORMSCALE_AXIS1_QNT_2D(I8,  F16, F16, vxc_char16,  vxc_char16,  vxc_half8,   half4,  vxc_short8)\n\
L2NORMSCALE_AXIS1_QNT_2D(I8,  F16, I8,  vxc_char16,  vxc_char16,  vxc_uchar16, int4,   vxc_char16)\n\
L2NORMSCALE_AXIS1_QNT_2D(I16, F16, F16, vxc_short8,  vxc_short8,  vxc_half8,   half4,  vxc_short8)\n\
L2NORMSCALE_AXIS1_QNT_2D(I16, F16, I16, vxc_short8,  vxc_short8,  vxc_short8,  int4,   vxc_short8)\n\
"; /* end of l2normalizescale_axis1_vx*/

static const char layer_normalization_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniSumX_16x1;\n\
_viv_uniform VXC_512Bits uniSumX2_16x1;\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_2_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_3_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int width;\n\
_viv_uniform float inv_multiplier;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define CONV2F32(dst, src, section) \\\n\
        VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
            uniDataToFP32_##section##_4x4);\n\
\n\
#define LAYER_NORM_8BITS_IMPL(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type src0, dst; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    uint2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.z, baseAddr); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = convert_float2(_sums) * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
    int2 coord_bias = (int2)(0, 0); \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1, scale, coord.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_bias.x = coord.x; \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
 \\\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        coord_bias.x += 4; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        coord_out.x = coord.x; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_8BITS_IMPL(U8_F16toU8, vxc_uchar16)\n\
LAYER_NORM_8BITS_IMPL(I8_F16toI8, vxc_char16)\n\
\n\
#define LAYER_NORM_SUMS_2D() \\\n\
    uint2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = convert_float2(_sums) * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y);\n\
\n\
#define LAYER_NORM_8BITS_IMPL_2D(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0); \\\n\
 \\\n\
    src_type src0, dst; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
 \\\n\
    LAYER_NORM_SUMS_2D(); \\\n\
 \\\n\
    int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
    int2 coord_bias = (int2)(0, 0); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1, scale, coord.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_bias.x = coord.x; \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
 \\\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        coord_bias.x += 4; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_8BITS_IMPL_2D(U8_F16toU8, vxc_uchar16)\n\
LAYER_NORM_8BITS_IMPL_2D(I8_F16toI8, vxc_char16)\n\
\n\
#define LAYER_NORM_8TOF16_IMPL(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    uint2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.z, baseAddr); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = convert_float2(_sums) * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
    int2 coord_bias = (int2)(0, 0); \\\n\
 \\\n\
    vxc_short8 dst; \\\n\
    vxc_half8 result; \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1, scale, coord.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_bias.x = coord.x; \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
 \\\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        coord_bias.x += 4; \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        coord_out.x = coord.x; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        coord_out.x += 8; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_8TOF16_IMPL(U8_F16toF16, vxc_uchar16)\n\
LAYER_NORM_8TOF16_IMPL(I8_F16toF16, vxc_char16)\n\
\n\
#define LAYER_NORM_8TOF16_IMPL_2D(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0); \\\n\
 \\\n\
    src_type src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
 \\\n\
    LAYER_NORM_SUMS_2D(); \\\n\
 \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
    int2 coord_bias = (int2)(0, 0); \\\n\
 \\\n\
    vxc_short8 dst; \\\n\
    vxc_half8 result; \\\n\
    for (coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1, scale, coord.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_bias.x = coord.x; \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
 \\\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        coord_bias.x += 4; \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x -= 8; \\\n\
    } \\\n\
}\n\
LAYER_NORM_8TOF16_IMPL_2D(U8_F16toF16, vxc_uchar16)\n\
LAYER_NORM_8TOF16_IMPL_2D(I8_F16toF16, vxc_char16)\n\
"; /* end of layer_normalization_0_vx*/

static const char layer_normalization_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniSum_X_X2_8x2;\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int width;\n\
_viv_uniform float inv_multiplier;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define CONV2F32(dst, src, section) \\\n\
        VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
            uniDataToFP32_##section##_4x4);\n\
\n\
#define LAYER_NORM_16BITS_IMPL(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel void layer_norm_axis0_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    vxc_short8 in0; \\\n\
    src_type src0; \\\n\
    copy_type dst; \\\n\
    vxc_short8 src1; \\\n\
    dst_type result; \\\n\
    vxc_half8 scale_h; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    float2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.z, baseAddr); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        coord.x += 8; \\\n\
        VXC_DP8x2(sum_x_x2, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniSum_X_X2_8x2); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = _sums * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
    int2 coord_bias = (int2)(0, 0); \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        VXC_ReadImage(src1, scale, coord.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_bias.x = coord.x; \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
 \\\n\
        coord_out.x = coord.x; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_16BITS_IMPL(F16_F16toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
LAYER_NORM_16BITS_IMPL(F16_F16toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16BITS_IMPL(F16_F16toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
LAYER_NORM_16BITS_IMPL(F16_F16toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
LAYER_NORM_16BITS_IMPL(I16_F16toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16BITS_IMPL(I16_F16toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
\n\
#define LAYER_NORM_16BITS_IMPL_2D(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel void layer_norm_axis0_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0); \\\n\
 \\\n\
    vxc_short8 in0; \\\n\
    src_type src0; \\\n\
    copy_type dst; \\\n\
    dst_type result; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    float2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_ReadImage(in0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        coord.x += 8; \\\n\
        VXC_DP8x2(sum_x_x2, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniSum_X_X2_8x2); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = _sums * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
 \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
    int2 coord_bias = (int2)(0, 0); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_ReadImage(in0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        VXC_ReadImage(src1, scale, coord.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_bias.x = coord.x; \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        CONV2F32(scale_f0, scale_h, 0); \\\n\
        CONV2F32(scale_f1, scale_h, 1); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        coord_bias.x += 4; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_16BITS_IMPL_2D(F16_F16toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
LAYER_NORM_16BITS_IMPL_2D(F16_F16toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16BITS_IMPL_2D(F16_F16toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
LAYER_NORM_16BITS_IMPL_2D(F16_F16toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
LAYER_NORM_16BITS_IMPL_2D(I16_F16toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16BITS_IMPL_2D(I16_F16toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
\n\
#define LAYER_NORM_16_32_IMPL(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel void layer_norm_axis0_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    vxc_short8 in0; \\\n\
    src_type src0; \\\n\
    copy_type dst; \\\n\
    dst_type result; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    float2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.z, baseAddr); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        coord.x += 8; \\\n\
        VXC_DP8x2(sum_x_x2, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniSum_X_X2_8x2); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = _sums * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scale_ptr = (__global float*)img2.ptr; \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        bias_f0 = vload4(0, bias_ptr); \\\n\
        bias_f1 = vload4(1, bias_ptr); \\\n\
        scale_f0 = vload4(0, scale_ptr); \\\n\
        scale_f1 = vload4(1, scale_ptr); \\\n\
        bias_ptr += 8; \\\n\
        scale_ptr += 8; \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
 \\\n\
        coord_out.x = coord.x; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_16_32_IMPL(F16_F32toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
LAYER_NORM_16_32_IMPL(F16_F32toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16_32_IMPL(F16_F32toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
LAYER_NORM_16_32_IMPL(F16_F32toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
LAYER_NORM_16_32_IMPL(I16_F32toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16_32_IMPL(I16_F32toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
\n\
#define LAYER_NORM_16_32_IMPL_2D(name, src_type, dst_type, copy_type, conv_type) \\\n\
__kernel void layer_norm_axis0_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0); \\\n\
 \\\n\
    vxc_short8 in0; \\\n\
    src_type src0; \\\n\
    copy_type dst; \\\n\
    dst_type result; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    float2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_ReadImage(in0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        coord.x += 8; \\\n\
        VXC_DP8x2(sum_x_x2, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniSum_X_X2_8x2); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = _sums * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
 \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scale_ptr = (__global float*)img2.ptr; \\\n\
    for (coord.x = 0; coord.x < width; coord.x += 8) \\\n\
    { \\\n\
        VXC_ReadImage(in0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src0, in0, 16); \\\n\
        bias_f0 = vload4(0, bias_ptr); \\\n\
        bias_f1 = vload4(1, bias_ptr); \\\n\
        scale_f0 = vload4(0, scale_ptr); \\\n\
        scale_f1 = vload4(1, scale_ptr); \\\n\
        bias_ptr += 8; \\\n\
        scale_ptr += 8; \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(CONV_RTE, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_16_32_IMPL_2D(F16_F32toF16, vxc_half8,  vxc_half8,  vxc_short8, half4)\n\
LAYER_NORM_16_32_IMPL_2D(F16_F32toI16, vxc_half8,  vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16_32_IMPL_2D(F16_F32toI8,  vxc_half8,  vxc_char8,  vxc_char8,  int4)\n\
LAYER_NORM_16_32_IMPL_2D(F16_F32toU8,  vxc_half8,  vxc_uchar8, vxc_uchar8, int4)\n\
LAYER_NORM_16_32_IMPL_2D(I16_F32toI16, vxc_short8, vxc_short8, vxc_short8, int4)\n\
LAYER_NORM_16_32_IMPL_2D(I16_F32toF16, vxc_short8, vxc_half8,  vxc_short8, half4)\n\
"; /* end of layer_normalization_1_vx*/

static const char layer_normalization_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniSumX_16x1;\n\
_viv_uniform VXC_512Bits uniSumX2_16x1;\n\
_viv_uniform VXC_512Bits uniSum_X_X2_8x2;\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_2_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_3_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int width;\n\
_viv_uniform float inv_multiplier;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define CONV2F32(dst, src, section) \\\n\
        VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
            uniDataToFP32_##section##_4x4);\n\
\n\
#define LAYER_NORM_8_32_IMPL(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type src0, dst; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    uint2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.z, baseAddr); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = convert_float2(_sums) * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scale_ptr = (__global float*)img2.ptr; \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        bias_f0 = vload4(0, bias_ptr); \\\n\
        bias_f1 = vload4(1, bias_ptr); \\\n\
        scale_f0 = vload4(0, scale_ptr); \\\n\
        scale_f1 = vload4(1, scale_ptr); \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = vload4(2, bias_ptr); \\\n\
        scale_f0 = vload4(2, scale_ptr); \\\n\
 \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        bias_f1 = vload4(3, bias_ptr); \\\n\
        scale_f1 = vload4(3, scale_ptr); \\\n\
        bias_ptr += 16; \\\n\
        scale_ptr += 16; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        coord_out.x = coord.x; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_8_32_IMPL(U8_F32toU8, vxc_uchar16)\n\
LAYER_NORM_8_32_IMPL(I8_F32toI8, vxc_char16)\n\
\n\
#define LAYER_NORM_8_32_IMPL_2D(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0); \\\n\
 \\\n\
    src_type src0, dst; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    uint2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = convert_float2(_sums) * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    int4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scale_ptr = (__global float*)img2.ptr; \\\n\
    for (coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        bias_f0 = vload4(0, bias_ptr); \\\n\
        bias_f1 = vload4(1, bias_ptr); \\\n\
        scale_f0 = vload4(0, scale_ptr); \\\n\
        scale_f1 = vload4(1, scale_ptr); \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = vload4(2, bias_ptr); \\\n\
        scale_f0 = vload4(2, scale_ptr); \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
 \\\n\
        bias_f1 = vload4(3, bias_ptr); \\\n\
        scale_f1 = vload4(3, scale_ptr); \\\n\
        bias_ptr += 16; \\\n\
        scale_ptr += 16; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_8_32_IMPL_2D(U8_F32toU8, vxc_uchar16)\n\
LAYER_NORM_8_32_IMPL_2D(I8_F32toI8, vxc_char16)\n\
\n\
#define LAYER_NORM_8_32TOF16_IMPL(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    uint2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.z, baseAddr); \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = convert_float2(_sums) * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
 \\\n\
    vxc_short8 dst; \\\n\
    vxc_half8 result; \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scale_ptr = (__global float*)img2.ptr; \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        bias_f0 = vload4(0, bias_ptr); \\\n\
        bias_f1 = vload4(1, bias_ptr); \\\n\
        scale_f0 = vload4(0, scale_ptr); \\\n\
        scale_f1 = vload4(1, scale_ptr); \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = vload4(2, bias_ptr); \\\n\
        scale_f0 = vload4(2, scale_ptr); \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
 \\\n\
        bias_f1 = vload4(3, bias_ptr); \\\n\
        scale_f1 = vload4(3, scale_ptr); \\\n\
        bias_ptr += 16; \\\n\
        scale_ptr += 16; \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        coord_out.x = coord.x; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        coord_out.x += 8; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_8_32TOF16_IMPL(U8_F32toF16, vxc_uchar16)\n\
LAYER_NORM_8_32TOF16_IMPL(I8_F32toF16, vxc_char16)\n\
\n\
#define LAYER_NORM_8_32TOF16_IMPL_2D(name, src_type) \\\n\
__kernel void layer_norm_axis0_##name##_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       bias, \\\n\
    __read_only  image2d_t       scale, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0); \\\n\
 \\\n\
    src_type src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h; \\\n\
    float sum = 0, sqr = 0; \\\n\
    float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    uint2 _sums = 0, sum_x_x2; \\\n\
 \\\n\
    for (coord.x = 0; coord.x < width; ) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1); \\\n\
        VXC_DP16x1(sum_x_x2, src0, src0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1); \\\n\
        _sums = _sums + sum_x_x2; \\\n\
    } \\\n\
 \\\n\
    float2 sums = convert_float2(_sums) * inv_multiplier; \\\n\
 \\\n\
    sums.y = sums.y - sums.x * sums.x + eps; \\\n\
    sums.y = rsqrt(sums.y); \\\n\
    half4 tmpVal0, tmpVal1; \\\n\
    float4  tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
 \\\n\
    vxc_short8 dst; \\\n\
    vxc_half8 result; \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scale_ptr = (__global float*)img2.ptr; \\\n\
    for (coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        bias_f0 = vload4(0, bias_ptr); \\\n\
        bias_f1 = vload4(1, bias_ptr); \\\n\
        scale_f0 = vload4(0, scale_ptr); \\\n\
        scale_f1 = vload4(1, scale_ptr); \\\n\
        bias_ptr += 8; \\\n\
        scale_ptr += 8; \\\n\
 \\\n\
        CONV2F32(tmpData0, src0, 0); \\\n\
        CONV2F32(tmpData1, src0, 1); \\\n\
        CONV2F32(tmpData2, src0, 2); \\\n\
        CONV2F32(tmpData3, src0, 3); \\\n\
 \\\n\
        float4 norm; \\\n\
        tmpData0 = tmpData0 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData0 + bias_f0; \\\n\
        bias_f0 = vload4(2, bias_ptr); \\\n\
        scale_f0 = vload4(2, scale_ptr); \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData1 = tmpData1 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData1 + bias_f1; \\\n\
        bias_f1 = vload4(3, bias_ptr); \\\n\
        scale_f1 = vload4(3, scale_ptr); \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        tmpData2 = tmpData2 - sums.x; \\\n\
        norm = scale_f0 * sums.y * tmpData2 + bias_f0; \\\n\
        _viv_asm(CONV, tmpVal0, norm); \\\n\
 \\\n\
        tmpData3 = tmpData3 - sums.x; \\\n\
        norm = scale_f1 * sums.y * tmpData3 + bias_f1; \\\n\
        _viv_asm(CONV, tmpVal1, norm); \\\n\
        VXC_DP2x8(result, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
            uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, dst, result, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x -= 8; \\\n\
    } \\\n\
}\n\
LAYER_NORM_8_32TOF16_IMPL_2D(U8_F32toF16, vxc_uchar16)\n\
LAYER_NORM_8_32TOF16_IMPL_2D(I8_F32toF16, vxc_char16)\n\
"; /* end of layer_normalization_2_vx*/

static const char layer_normalization_3_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************layernorm float16***********************************/\n\
_viv_uniform int width;\n\
_viv_uniform float inv_multiplier;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void layer_norm_BF16F32toBF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 ones = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    vxc_ushort8 src0, src1, src2;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_OP4(img_load_3d, src0, input, coord.xyzz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
    float4 srcA, srcB;\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        sum += dot(srcA, ones) + dot(srcB, ones);\n\
        sqr += dot(srcA * srcA, ones) + dot(srcB * srcB, ones);\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * inv_multiplier;\n\
    vxc_float vari;\n\
    vari = sqr*inv_multiplier - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, (int2)(0, 0));\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, (int2)(0, 0));\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        bias_ptr += 8;\n\
        scale_ptr += 8;\n\
\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
\n\
\n\
        vxc_float4 sub0, sub1, norm0, norm1;\n\
        sub0 = srcA - mean;\n\
        sub1 = srcB - mean;\n\
        norm0 = scale_f0 * vari * sub0 + bias_f0;\n\
        norm1 = scale_f1 * vari * sub1 + bias_f1;\n\
\n\
        _viv_asm(COPY, src0, norm0, 16);\n\
        _viv_asm(COPY, src1, norm1, 16);\n\
        VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, src2, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void layer_norm_BF16F32toBF16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_ushort8 src0, src1, src2;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 ones = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    float4 srcA, srcB;\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        sum += dot(srcA, ones) + dot(srcB, ones);\n\
        sqr += dot(srcA * srcA, ones) + dot(srcB * srcB, ones);\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * inv_multiplier;\n\
    vxc_float vari;\n\
    vari = sqr*inv_multiplier - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, coord.zw);\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, coord.zw);\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        bias_ptr += 8;\n\
        scale_ptr += 8;\n\
\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
\n\
        vxc_float4 sub0, sub1, norm0, norm1;\n\
        sub0 = srcA - mean;\n\
        sub1 = srcB - mean;\n\
        norm0 = scale_f0 * vari * sub0 + bias_f0;\n\
        norm1 = scale_f1 * vari * sub1 + bias_f1;\n\
\n\
        _viv_asm(COPY, src0, norm0, 16);\n\
        _viv_asm(COPY, src1, norm1, 16);\n\
        VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_3_vx*/

static const char layer_normalization_axis01_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int height;\n\
_viv_uniform uint group_num;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float inv_multiplier;\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_U8_F16toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_uchar16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y; coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniExtract8Data_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_U8_F32toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_uchar16 src0;\n\
    vxc_short8 outval;\n\
    vxc_half8 scale_h, dst;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_bias.y = coord.y;\n\
\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniExtract8Data_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_U8_F16toU8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_uchar16 src0, outval;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y; coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_U8_F32toU8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_uchar16 src0 , outval;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_bias.y = coord.y;\n\
\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_axis01_0_vx*/

static const char layer_normalization_axis01_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int height;\n\
_viv_uniform uint group_num;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float inv_multiplier;\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_I8_F16toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_char16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y; coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniExtract8Data_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_I8_F32toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_char16 src0;\n\
    vxc_short8 outval;\n\
    vxc_half8 scale_h, dst;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_bias.y = coord.y;\n\
\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniExtract8Data_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_I8_F16toU8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_char16 src0;\n\
    vxc_uchar16 outval;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y; coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_I8_F32toU8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    vxc_char16 src0;\n\
    vxc_uchar16 outval;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    coord_para.z = 0;\n\
    coord_para.w = 0;\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_bias.y = coord.y;\n\
\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4);\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4);\n\
        tmpData0 = tmpData0 - mean_vari.s0;\n\
        tmpData1 = tmpData1 - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_axis01_1_vx*/

static const char layer_normalization_axis01_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int height;\n\
_viv_uniform uint group_num;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float inv_multiplier;\n\
\n\
#define LAYER_NORM_AXIS01_F16_F16to16Bits(name,temp_type,dst_type,conv_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_F16_F16to##name( \\\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_sum = (int2)(0, gidz); \\\n\
    int4 coord_para = coord; \\\n\
    vxc_short8 src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h, in_h; \\\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += read_imagef(meanVari, coord_sum); \\\n\
        coord_sum.x += 4; \\\n\
    } \\\n\
    mean_vari *= inv_multiplier; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    coord_para.z = 0; \\\n\
    coord_para.w = 0; \\\n\
    int4 coord_bias = coord_para; \\\n\
 \\\n\
    int8 input_desc, scale_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc)); \\\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0; \\\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    vxc_float4  tmpData0, tmpData1; \\\n\
    vxc_short8 outval; \\\n\
    temp_type tmpVal0, tmpVal1; \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y ++; \\\n\
        coord_para.y = coord.y; \\\n\
        coord_bias.y = coord.y; \\\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x = coord.x; \\\n\
 \\\n\
        _viv_asm(COPY, in_h, src0, 16); \\\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_1_4x4); \\\n\
 \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4); \\\n\
 \\\n\
        vxc_float4 sub, norm; \\\n\
        sub = tmpData0 - mean_vari.s0; \\\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal0, norm); \\\n\
        sub = tmpData1 - mean_vari.s0; \\\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal1, norm); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, outval, dst, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_AXIS01_F16_F16to16Bits(F16,half4,vxc_half8,CONV)\n\
LAYER_NORM_AXIS01_F16_F16to16Bits(I16,int4,vxc_short8,CONV_RTE)\n\
\n\
\n\
#define LAYER_NORM_AXIS01_F16_F32to16Bits(name,temp_type,dst_type,conv_type)  \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_F16_F32to##name( \\\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_sum = (int2)(0, gidz); \\\n\
    int4 coord_para = coord; \\\n\
    vxc_short8 src0; \\\n\
    vxc_half8 in_h; \\\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += read_imagef(meanVari, coord_sum); \\\n\
        coord_sum.x += 4; \\\n\
    } \\\n\
    mean_vari *= inv_multiplier; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    coord_para.z = 0; \\\n\
    coord_para.w = 0; \\\n\
    int4 coord_bias = coord_para; \\\n\
 \\\n\
    int8 input_desc, scale_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    vxc_float4  tmpData0, tmpData1; \\\n\
    vxc_short8 outval; \\\n\
    temp_type tmpVal0, tmpVal1; \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y ++; \\\n\
        coord_bias.y = coord.y; \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        scale_f0 = read_imagef(scale, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        scale_f1 = read_imagef(scale, coord_bias); \\\n\
        coord_bias.x = coord.x; \\\n\
 \\\n\
        _viv_asm(COPY, in_h, src0, 16); \\\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_1_4x4); \\\n\
 \\\n\
        vxc_float4 sub, norm; \\\n\
        sub = tmpData0 - mean_vari.s0; \\\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal0, norm); \\\n\
        sub = tmpData1 - mean_vari.s0; \\\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal1, norm); \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, outval, dst, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_AXIS01_F16_F32to16Bits(F16,half4,vxc_half8,CONV)\n\
LAYER_NORM_AXIS01_F16_F32to16Bits(I16,int4,vxc_short8,CONV_RTE)\n\
\n\
#define LAYER_NORM_AXIS01_F16_F16toQUANT(name,dst_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_F16_F16to##name( \\\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_sum = (int2)(0, gidz); \\\n\
    int4 coord_para = coord; \\\n\
    vxc_short8 src0; \\\n\
    vxc_short8 src1; \\\n\
    vxc_half8 scale_h, in_h; \\\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += read_imagef(meanVari, coord_sum); \\\n\
        coord_sum.x += 4; \\\n\
    } \\\n\
    mean_vari *= inv_multiplier; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    coord_para.z = 0; \\\n\
    coord_para.w = 0; \\\n\
    int4 coord_bias = coord_para; \\\n\
 \\\n\
    int8 input_desc, scale_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc)); \\\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0; \\\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    vxc_float4  tmpData0, tmpData1; \\\n\
    dst_type outval; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y ++; \\\n\
        coord_para.y = coord.y; \\\n\
        coord_bias.y = coord.y; \\\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x = coord.x; \\\n\
 \\\n\
        _viv_asm(COPY, in_h, src0, 16); \\\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_1_4x4); \\\n\
 \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4); \\\n\
 \\\n\
        vxc_float4 sub, norm; \\\n\
        sub = tmpData0 - mean_vari.s0; \\\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        sub = tmpData1 - mean_vari.s0; \\\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_AXIS01_F16_F16toQUANT(U8,vxc_uchar16)\n\
LAYER_NORM_AXIS01_F16_F16toQUANT(I8,vxc_char16)\n\
\n\
#define LAYER_NORM_AXIS01_F16_F32toQUANT(name,dst_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_F16_F32to##name( \\\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_sum = (int2)(0, gidz); \\\n\
    int4 coord_para = coord; \\\n\
    vxc_short8 src0; \\\n\
    vxc_half8 in_h; \\\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += read_imagef(meanVari, coord_sum); \\\n\
        coord_sum.x += 4; \\\n\
    } \\\n\
    mean_vari *= inv_multiplier; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    coord_para.z = 0; \\\n\
    coord_para.w = 0; \\\n\
    int4 coord_bias = coord_para; \\\n\
 \\\n\
    int8 input_desc, scale_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    vxc_float4  tmpData0, tmpData1; \\\n\
    dst_type outval; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y ++; \\\n\
        coord_bias.y = coord.y; \\\n\
 \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        scale_f0 = read_imagef(scale, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        scale_f1 = read_imagef(scale, coord_bias); \\\n\
        coord_bias.x = coord.x; \\\n\
 \\\n\
        _viv_asm(COPY, in_h, src0, 16); \\\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataToFP32_1_4x4); \\\n\
 \\\n\
        vxc_float4 sub, norm; \\\n\
        sub = tmpData0 - mean_vari.s0; \\\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0; \\\n\
        tmpVal0 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        sub = tmpData1 - mean_vari.s0; \\\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1; \\\n\
        tmpVal1 = convert_int4_rte(norm * output_scale + output_zp); \\\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_AXIS01_F16_F32toQUANT(U8,vxc_uchar16)\n\
LAYER_NORM_AXIS01_F16_F32toQUANT(I8,vxc_char16)"; /* end of layer_normalization_axis01_2_vx*/

static const char layer_normalization_axis01_3_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataToFP32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataToFP32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int height;\n\
_viv_uniform uint group_num;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float inv_multiplier;\n\
\n\
#define LAYER_NORM_AXIS01_I16_F16to16Bits(name,temp_type,dst_type,conv_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_I16_F16to##name( \\\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_sum = (int2)(0, gidz); \\\n\
    int4 coord_para = coord; \\\n\
    vxc_short8 src0, src1; \\\n\
    vxc_half8 scale_h; \\\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += read_imagef(meanVari, coord_sum); \\\n\
        coord_sum.x += 4; \\\n\
    } \\\n\
    mean_vari *= inv_multiplier; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    coord_para.z = 0; \\\n\
    coord_para.w = 0; \\\n\
    int4 coord_bias = coord_para; \\\n\
 \\\n\
    int8 input_desc, scale_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc)); \\\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0; \\\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    vxc_float4  tmpData0, tmpData1, norm; \\\n\
    temp_type tmpVal0, tmpVal1; \\\n\
    vxc_short8 outval; \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y ++; \\\n\
        coord_para.y = coord.y; \\\n\
        coord_bias.y = coord.y; \\\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        coord_bias.x = coord.x; \\\n\
 \\\n\
        _viv_asm(COPY, scale_h, src1, 16); \\\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4); \\\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4); \\\n\
        tmpData0 = tmpData0 - mean_vari.s0; \\\n\
        tmpData1 = tmpData1 - mean_vari.s0; \\\n\
 \\\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal0, norm); \\\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal1, norm); \\\n\
 \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, outval, dst, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_AXIS01_I16_F16to16Bits(F16,half4,vxc_half8,CONV)\n\
LAYER_NORM_AXIS01_I16_F16to16Bits(I16,int4,vxc_short8,CONV_RTE)\n\
\n\
\n\
#define LAYER_NORM_AXIS01_I16_F32to16Bits(name,temp_type,dst_type,conv_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_I16_F32to##name( \\\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_sum = (int2)(0, gidz); \\\n\
    int4 coord_para = coord; \\\n\
    vxc_short8 src0; \\\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += read_imagef(meanVari, coord_sum); \\\n\
        coord_sum.x += 4; \\\n\
    } \\\n\
    mean_vari *= inv_multiplier; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    coord_para.z = 0; \\\n\
    coord_para.w = 0; \\\n\
    int4 coord_bias = coord_para; \\\n\
 \\\n\
    int8 input_desc, scale_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
 \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    vxc_float4  tmpData0, tmpData1, norm; \\\n\
    temp_type tmpVal0, tmpVal1; \\\n\
    vxc_short8 outval; \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y ++; \\\n\
        coord_bias.y = coord.y; \\\n\
        bias_f0 = read_imagef(bias, coord_bias); \\\n\
        scale_f0 = read_imagef(scale, coord_bias); \\\n\
        coord_bias.x += 4; \\\n\
        bias_f1 = read_imagef(bias, coord_bias); \\\n\
        scale_f1 = read_imagef(scale, coord_bias); \\\n\
        coord_bias.x = coord.x; \\\n\
 \\\n\
        VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_0_4x4); \\\n\
        VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniDataToFP32_1_4x4); \\\n\
        tmpData0 = tmpData0 - mean_vari.s0; \\\n\
        tmpData1 = tmpData1 - mean_vari.s0; \\\n\
 \\\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal0, norm); \\\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1; \\\n\
        norm = norm * output_scale + output_zp; \\\n\
        _viv_asm(conv_type, tmpVal1, norm); \\\n\
 \\\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniExtract8Data_2x8); \\\n\
        _viv_asm(COPY, outval, dst, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
LAYER_NORM_AXIS01_I16_F32to16Bits(F16,half4,vxc_half8,CONV)\n\
LAYER_NORM_AXIS01_I16_F32to16Bits(I16,int4,vxc_short8,CONV_RTE)\n\
"; /* end of layer_normalization_axis01_3_vx*/

static const char layer_normalization_axis01_sum_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniSumX_16x1;\n\
_viv_uniform VXC_512Bits uniSumX2_16x1;\n\
_viv_uniform VXC_512Bits uniSum_X_X2_8x2;\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_sums_F16toF32(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniSum_X_X2_8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0;\n\
        float sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_sums_I16toF32(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    float4 tmpSumSqr = (float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            vxc_float4 sumsqr;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniSum_X_X2_8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        float4 data = (float4)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            data.x += dot(tmp_sum[i], one);\n\
            data.y += dot(tmp_sqr[i], one);\n\
        }\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_sums_U8toF32(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord.xywz, 0,\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1);\n\
            tmpSqr += (tmpSqr1);\n\
        }\n\
        sqr += convert_float(tmpSqr);\n\
        sum = convert_float(tmpSum);\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_axis01_sums_I8toF32(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_char16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord.xywz, 0,\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumX2_16x1);\n\
            tmpSqr += (tmpSqr1);\n\
        }\n\
        sqr += convert_float(tmpSqr);\n\
        sum = convert_float(tmpSum);\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
"; /* end of layer_normalization_axis01_sum_vx*/

static const char log_softmax_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform int         inputWidthRemain4;\n\
_viv_uniform VXC_512Bits uniGetSubData0to3_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubData4to7_4x4;\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0(read_fun, vert_max_fun, horz_max_fun) \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        vert_max_fun(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        vert_max_fun(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16)); \\\n\
    horz_max_fun(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    horz_max_fun(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_float4 prob; \\\n\
    float fProbSum = 0; \\\n\
    const float4 one4 = (float4)(1.0, 1.0, 1.0, 1.0); \\\n\
    int idx = 0; \\\n\
    for (coord.x = 0; coord.x < inputWidth; idx ++) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
        prob *= scaleLogE; \\\n\
        prob = exp2(prob); \\\n\
        fProbSum += dot(prob, one4); \\\n\
        coord.x += 4; \\\n\
    } \\\n\
    read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, img_val0, val0, 16); \\\n\
    VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
    prob *= scaleLogE; \\\n\
    if(inputWidthRemain4 == 1) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.yzw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 2) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.zw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 3) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.z = exp2(prob.z); \\\n\
        prob.w = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    vxc_float4 probSum_log; \\\n\
    probSum_log.x = log2(fProbSum) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        dst_type vec0, vec1; \\\n\
        save_type dst; \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        prob = prob * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, vec0, prob); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        prob = prob * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, vec1, prob); \\\n\
        VXC_DP2x8(dst, vec0, vec1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
        write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
                        save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                 CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage, VXC_WriteImage); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_TOF32_SAVE(read_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        write_imagef(output, coord, prob); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        write_imagef(output, coord, prob); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_AXIS0_TOF32(src_name, src_type, copy_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_TOF32_SAVE(VXC_ReadImage2DArray) \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0_TOF32(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_TOF32(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS0_TOF32_2D(src_name, src_type, copy_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_t       output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_TOF32_SAVE(VXC_ReadImage) \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0_TOF32_2D(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_TOF32_2D(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32_2D(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32_2D(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
"; /* end of log_softmax_axis0_vx*/

static const char log_softmax_axis0_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform int         inputWidthRemain4;\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16(read_fun) \\\n\
    vxc_half8 img_val0, img_val1, img_val2, img_val3; \\\n\
    vxc_short8 val0, val1, val2, val3; \\\n\
    vxc_half8 val; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        VXC_VertMax3_Half(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_VertMax3_Half(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16)); \\\n\
    VXC_HorzMax3_Half(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    VXC_HorzMax3_Half(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_ushort8   bf_val_tmp; \\\n\
    vxc_float4 vecA; \\\n\
    _viv_asm(COPY, bf_val_tmp, val, 16); \\\n\
    VXC_DP2x8(bf_val_tmp, bf_val_tmp, zero,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, bf_val_tmp, 16); \\\n\
    vxc_float4 prob; \\\n\
    float fProbSum = 0; \\\n\
    const float4 one4 = (float4)(1.0, 1.0, 1.0, 1.0); \\\n\
    float max_value = vecA.x * scaleLogE; \\\n\
    float max_value_orig = vecA.x; \\\n\
    for (coord.x = 0; coord.x < inputWidth; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob * scaleLogE - max_value; \\\n\
        prob = exp2(prob); \\\n\
        fProbSum += dot(prob, one4); \\\n\
        coord.x += 4; \\\n\
    } \\\n\
    read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
    prob = prob * scaleLogE - max_value; \\\n\
    if(inputWidthRemain4 == 1) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.yzw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 2) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.zw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 3) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.z = exp2(prob.z); \\\n\
        prob.w = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    vxc_float4 probSum_log; \\\n\
    probSum_log.x = log2(fProbSum) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16TOBF16_SAVE(read_fun, write_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob - max_value_orig; \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        vxc_ushort8 tmp, dst; \\\n\
        _viv_asm(COPY, tmp, prob, 16); \\\n\
        dst.s0123 = tmp.s1357; \\\n\
        write_fun(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16TOF16_SAVE(read_fun, write_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob - max_value_orig; \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        half4 vec; \\\n\
        vxc_half4 tmp; \\\n\
        vxc_short4 dst; \\\n\
        _viv_asm(CONV, vec, prob); \\\n\
        VXC_DP4x4(tmp, vec, vec, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
        _viv_asm(COPY, dst, tmp, 8); \\\n\
        write_fun(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16TOF32_SAVE(read_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob - max_value_orig; \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        write_imagef(output, coord, prob); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
__kernel void log_softmax_axis0_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOBF16_SAVE(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF16_SAVE(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF32_SAVE(VXC_ReadImage2DArray)\n\
}\n\
__kernel void log_softmax_axis0_BF16toBF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(16, get_global_id(0));\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOBF16_SAVE(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(16, get_global_id(0));\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF16_SAVE(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF32_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t        output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(16, get_global_id(0));\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF32_SAVE(VXC_ReadImage)\n\
}\n\
\n\
"; /* end of log_softmax_axis0_BF16_vx*/

static const char log_softmax_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGetSubLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubHiData_4x4;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS1(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
    coord.y = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum0 += data0; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum1 += data0; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
                 OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    coord.y = 0; \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst0, data0); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst1, data0); \\\n\
        VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
        write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define LOGSOFTMAX_AXIS1(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
\n\
\n\
LOGSOFTMAX_AXIS1(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half)\n\
\n\
LOGSOFTMAX_AXIS1(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8, CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS1_2D(src_name, dst_name, src_type,\\\n\
copy_type, dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage, VXC_WriteImage); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS1_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half)\n\
\n\
LOGSOFTMAX_AXIS1_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8, CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
\n\
#define LOGSOFTMAX_AXIS1_TOF32(src_name, src_type, copy_type, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    coord.y = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x -= 4; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS1_TOF32(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_TOF32(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS1_TOF32_2D(src_name, src_type, copy_type, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_t       output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    coord.y = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x -= 4; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS1_TOF32_2D(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_TOF32_2D(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32_2D(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32_2D(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer)\n\
"; /* end of log_softmax_axis1_vx*/

static const char log_softmax_axis1_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS1_BF16(read_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_VertMax3_Half(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
    _viv_asm(COPY, tmp0, max, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, max_lo, tmp1, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, max_hi, tmp1, 16); \\\n\
    coord.y = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0 = data0 - max_lo; \\\n\
        data0 *= scaleLogE; \\\n\
        sum0  += exp2(data0); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0 = data0 - max_hi; \\\n\
        data0 *= scaleLogE; \\\n\
        sum1  += exp2(data0); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while (coord.y < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
__kernel void log_softmax_axis1_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(COPY, dst0, data0, 16);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(COPY, dst1, data0, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    half4 dst0, dst1;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(CONV, dst0, data0);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(CONV, dst1, data0);\n\
        VXC_DP2x8(vec0, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
        vxc_short8 vect;\n\
        _viv_asm(COPY, vect, vec0, 16);\n\
        VXC_WriteImage2DArray(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        write_imagef(output, coord, data0);\n\
        coord.x += 4;\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        write_imagef(output, coord, data0);\n\
        coord.x -= 4;\n\
        coord.y++;\n\
    }\n\
    while (coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toBF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
\n\
    coord.y = 0;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    do\n\
    {\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(COPY, dst0, data0, 16);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(COPY, dst1, data0, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
\n\
    coord.y = 0;\n\
    half4 dst0, dst1;\n\
    do\n\
    {\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(CONV, dst0, data0);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(CONV, dst1, data0);\n\
        VXC_DP2x8(vec0, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
        vxc_short8 vect;\n\
        _viv_asm(COPY, vect, vec0, 16);\n\
        VXC_WriteImage(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF32_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t        output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
\n\
    coord.y = 0;\n\
    do\n\
    {\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        write_imagef(output, coord, data0);\n\
        coord.x += 4;\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        write_imagef(output, coord, data0);\n\
        coord.x -= 4;\n\
        coord.y++;\n\
    }\n\
    while (coord.y < axisSize);\n\
}\n\
"; /* end of log_softmax_axis1_BF16_vx*/

static const char log_softmax_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGetSubLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubHiData_4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS2(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
    coord.z = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum0 += data0; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum1 += data0; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS2_SAVE(dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    coord.z = 0; \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst0, data0); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst1, data0); \\\n\
        VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
        write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
#define LOGSOFTMAX_AXIS2(src_name, dst_name, src_type, copy_type,\\\n\
dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun) \\\n\
__kernel void log_softmax_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS2(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half)\n\
\n\
LOGSOFTMAX_AXIS2(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS2(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
\n\
#define LOGSOFTMAX_AXIS2_TOF32(src_name, src_type, copy_type, vert_max_fun) \\\n\
__kernel void log_softmax_axis2_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    coord.z = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x -= 4; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS2_TOF32(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2_TOF32(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2_TOF32(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2_TOF32(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer)\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS2_BF16(read_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_VertMax3_Half(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
    _viv_asm(COPY, tmp0, max, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, max_lo, tmp1, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, max_hi, tmp1, 16); \\\n\
    coord.z = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0  = data0 - max_lo; \\\n\
        data0 *= scaleLogE; \\\n\
        sum0  += exp2(data0); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0  = data0 - max_hi; \\\n\
        data0 *= scaleLogE; \\\n\
        sum1  += exp2(data0); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while (coord.z < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
__kernel void log_softmax_axis2_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.z = 0;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(COPY, dst0, data0, 16);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(COPY, dst1, data0, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
    while(coord.z < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis2_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.z = 0;\n\
    half4 dst0, dst1;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(CONV, dst0, data0);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(CONV, dst1, data0);\n\
        VXC_DP2x8(vec0, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
        vxc_short8 vect;\n\
        _viv_asm(COPY, vect, vec0, 16);\n\
        VXC_WriteImage2DArray(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
    while(coord.z < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis2_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.z = 0;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        write_imagef(output, coord, data0);\n\
        coord.x += 4;\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        write_imagef(output, coord, data0);\n\
        coord.x -= 4;\n\
        coord.z++;\n\
    }\n\
    while (coord.z < axisSize);\n\
}\n\
"; /* end of log_softmax_axis2_vx*/

static const char log_softmax_exceed_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform int         height;\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform int         inputWidthRemain4;\n\
_viv_uniform VXC_512Bits uniGetSubData0to3_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubData4to7_4x4;\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS0(read_fun, vert_max_fun, horz_max_fun) \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 16;coord.x < (axisSize + 16);coord.x+=32) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), \\\n\
                                          VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val0, val0, 16); \\\n\
            read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), \\\n\
                                          VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val1, val1, 16); \\\n\
            read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val2, val2, 16); \\\n\
            read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
                                          VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val3, val3, 16); \\\n\
            vert_max_fun(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            vert_max_fun(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    horz_max_fun(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    horz_max_fun(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_float4 prob; \\\n\
    float fProbSum = 0; \\\n\
    const float4 one4 = (float4)(1.0, 1.0, 1.0, 1.0); \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 0;coord.x < inputWidth;coord.x+=4) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val0, val0, 16); \\\n\
            VXC_DP4x4(prob, img_val0, val,\\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
            prob *= scaleLogE; \\\n\
            prob = exp2(prob); \\\n\
            fProbSum += dot(prob, one4); \\\n\
        } \\\n\
    } \\\n\
    read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, img_val0, val0, 16); \\\n\
    VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
    prob *= scaleLogE; \\\n\
    if(inputWidthRemain4 == 1) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.yzw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 2) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.zw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 3) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.z = exp2(prob.z); \\\n\
        prob.w = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    vxc_float4 probSum_log; \\\n\
    probSum_log.x = log2(fProbSum) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS0_SAVE(dst_type, \\\n\
        save_type, conv_mode, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 0; coord.x < axisSize; coord.x += 8) \\\n\
        { \\\n\
            dst_type vec0, vec1; \\\n\
            save_type dst; \\\n\
            read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val0, val0, 16); \\\n\
            VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
            prob = prob * betaValue - probSum_log.xxxx; \\\n\
            prob = prob * OUT_SCALE + OUT_OFFSET; \\\n\
            _viv_asm(conv_mode, vec0, prob); \\\n\
            VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4); \\\n\
            prob = prob * betaValue - probSum_log.xxxx; \\\n\
            prob = prob * OUT_SCALE + OUT_OFFSET; \\\n\
            _viv_asm(conv_mode, vec1, prob); \\\n\
            VXC_DP2x8(dst, vec0, vec1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
            write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    }\n\
\n\
#define LOGSOFTMAX_EXCEED_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
                        save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_exceed_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, 0, get_global_id(1), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
LOGSOFTMAX_EXCEED_AXIS0(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                 CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS0(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS0(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS0_TOF32_SAVE(read_fun) \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 0; coord.x < axisSize; ) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val0, val0, 16); \\\n\
            VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
            prob = prob * betaValue - probSum_log.xxxx; \\\n\
            write_imagef(output, coord, prob); \\\n\
            coord.x += 4; \\\n\
            VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4); \\\n\
            prob = prob * betaValue - probSum_log.xxxx; \\\n\
            write_imagef(output, coord, prob); \\\n\
            coord.x += 4; \\\n\
        } \\\n\
    }\n\
\n\
#define LOGSOFTMAX_EXCEED_AXIS0_TOF32(src_name, src_type, copy_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_exceed_axis0_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, 0, get_global_id(1), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_TOF32_SAVE(VXC_ReadImage2DArray) \\\n\
}\n\
\n\
LOGSOFTMAX_EXCEED_AXIS0_TOF32(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS0_TOF32(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS0_TOF32(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS0_TOF32(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
"; /* end of log_softmax_exceed_axis0_vx*/

static const char log_softmax_exceed_axis0_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform int         height;\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform int         inputWidthRemain4;\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16(read_fun) \\\n\
    vxc_half8 img_val0, img_val1, img_val2, img_val3; \\\n\
    vxc_short8 val0, val1, val2, val3; \\\n\
    vxc_half8 val; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 16; coord.x < (axisSize + 16);) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), \\\n\
                                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val0, val0, 16); \\\n\
            read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), \\\n\
                                  VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val1, val1, 16); \\\n\
            read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val2, val2, 16); \\\n\
            read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
                                  VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, img_val3, val3, 16); \\\n\
            coord.x += 32; \\\n\
            VXC_VertMax3_Half(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_VertMax3_Half(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    VXC_HorzMax3_Half(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    VXC_HorzMax3_Half(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_ushort8   bf_val_tmp; \\\n\
    vxc_float4 vecA; \\\n\
    _viv_asm(COPY, bf_val_tmp, val, 16); \\\n\
    VXC_DP2x8(bf_val_tmp, bf_val_tmp, zero,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, bf_val_tmp, 16); \\\n\
    vxc_float4 prob; \\\n\
    float fProbSum = 0; \\\n\
    const float4 one4 = (float4)(1.0, 1.0, 1.0, 1.0); \\\n\
    float max_value = vecA.x * scaleLogE; \\\n\
    float max_value_orig = vecA.x; \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 0; coord.x < inputWidth; ) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
            _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
            prob = prob * scaleLogE - max_value; \\\n\
            prob = exp2(prob); \\\n\
            fProbSum += dot(prob, one4); \\\n\
            coord.x += 4; \\\n\
        } \\\n\
    } \\\n\
    read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
    prob = prob * scaleLogE - max_value; \\\n\
    if(inputWidthRemain4 == 1) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.yzw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 2) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.zw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 3) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.z = exp2(prob.z); \\\n\
        prob.w = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    vxc_float4 probSum_log; \\\n\
    probSum_log.x = log2(fProbSum) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16TOBF16_SAVE(read_fun, write_fun) \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 0; coord.x < axisSize; ) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
            _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
            prob = prob - max_value_orig; \\\n\
            prob = prob * betaValue - probSum_log.xxxx; \\\n\
            vxc_ushort8 tmp, dst; \\\n\
            _viv_asm(COPY, tmp, prob, 16); \\\n\
            dst.s0123 = tmp.s1357; \\\n\
            write_fun(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.x += 4; \\\n\
        } \\\n\
    }\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16TOF16_SAVE(read_fun, write_fun) \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 0; coord.x < axisSize; ) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
            _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
            prob = prob - max_value_orig; \\\n\
            prob = prob * betaValue - probSum_log.xxxx; \\\n\
            half4 vec; \\\n\
            vxc_half4 tmp; \\\n\
            vxc_short4 dst; \\\n\
            _viv_asm(CONV, vec, prob); \\\n\
            VXC_DP4x4(tmp, vec, vec, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
            _viv_asm(COPY, dst, tmp, 8); \\\n\
            write_fun(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.x += 4; \\\n\
        } \\\n\
    }\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16TOF32_SAVE(read_fun) \\\n\
    for (coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
        for (coord.x = 0; coord.x < axisSize; ) \\\n\
        { \\\n\
            read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
            _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
            prob = prob - max_value_orig; \\\n\
            prob = prob * betaValue - probSum_log.xxxx; \\\n\
            write_imagef(output, coord, prob); \\\n\
            coord.x += 4; \\\n\
        } \\\n\
    }\n\
\n\
__kernel void log_softmax_exceed_axis0_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, 0, get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16TOBF16_SAVE(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
__kernel void log_softmax_exceed_axis0_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, 0, get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16TOF16_SAVE(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
__kernel void log_softmax_exceed_axis0_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, 0, get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS0_BF16TOF32_SAVE(VXC_ReadImage2DArray)\n\
}\n\
\n\
"; /* end of log_softmax_exceed_axis0_BF16_vx*/

static const char log_softmax_exceed_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         depth;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGetSubLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubHiData_4x4;\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS1(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    for (coord.z = 0; coord.z < depth; coord.z ++) \\\n\
    { \\\n\
        for (coord.y = 0; coord.y < axisSize;) \\\n\
        { \\\n\
            read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, vec0, in0, 16); \\\n\
            vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
        } \\\n\
    } \\\n\
    coord.y = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    for (coord.z = 0; coord.z < depth; coord.z ++) \\\n\
    { \\\n\
        for (coord.y = 0; coord.y < axisSize;coord.y++) \\\n\
        { \\\n\
            read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, vec0, in0, 16); \\\n\
            VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, \\\n\
                                         VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
            data0 *= scaleLogE; \\\n\
            data0 = exp2(data0); \\\n\
            sum0 += data0; \\\n\
            VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, \\\n\
                                         VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
            data0 *= scaleLogE; \\\n\
            data0 = exp2(data0); \\\n\
            sum1 += data0; \\\n\
        } \\\n\
    } \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
                 OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    coord.y = 0; \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    for (coord.z = 0; coord.z < depth; coord.z ++) \\\n\
    { \\\n\
        for (coord.y = 0; coord.y < axisSize;) \\\n\
        { \\\n\
            read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, vec0, in0, 16); \\\n\
            VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, \\\n\
                                              VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
            data0 = data0 * betaValue - sum0; \\\n\
            data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
            _viv_asm(conv_mode, dst0, data0); \\\n\
            VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, \\\n\
                                               VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
            data0 = data0 * betaValue - sum1; \\\n\
            data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
            _viv_asm(conv_mode, dst1, data0); \\\n\
            VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, \\\n\
                                          VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
            write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
        } \\\n\
    } \\\n\
\n\
#define LOGSOFTMAX_EXCEED_AXIS1(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun) \\\n\
__kernel void log_softmax_exceed_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
\n\
\n\
LOGSOFTMAX_EXCEED_AXIS1(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half)\n\
\n\
LOGSOFTMAX_EXCEED_AXIS1(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_EXCEED_AXIS1(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_EXCEED_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4, \\\n\
vxc_short8, CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
\n\
\n\
#define LOGSOFTMAX_EXCEED_AXIS1_TOF32(src_name, src_type, copy_type, vert_max_fun) \\\n\
__kernel void log_softmax_exceed_axis1_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    coord.y = 0; \\\n\
    for (coord.z = 0; coord.z < depth; coord.z ++) \\\n\
    { \\\n\
        for (coord.y = 0; coord.y < axisSize;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, vec0, in0, 16); \\\n\
            VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, \\\n\
                                  VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
            data0 = data0 * betaValue - sum0; \\\n\
            write_imagef(output, coord, data0); \\\n\
            coord.x += 4; \\\n\
            VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, \\\n\
                                   VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
            data0 = data0 * betaValue - sum1; \\\n\
            write_imagef(output, coord, data0); \\\n\
            coord.x -= 4; \\\n\
            coord.y++; \\\n\
        } \\\n\
    } \\\n\
}\n\
\n\
LOGSOFTMAX_EXCEED_AXIS1_TOF32(F16, vxc_half8, \\\n\
vxc_short8, VXC_VertMax3_Half)\n\
LOGSOFTMAX_EXCEED_AXIS1_TOF32(I16, vxc_short8, \\\n\
vxc_short8, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS1_TOF32(I8,  vxc_char16, \\\n\
vxc_char16, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_EXCEED_AXIS1_TOF32(U8,  vxc_uchar16, \\\n\
vxc_uchar16, VXC_VertMax3_Integer)"; /* end of log_softmax_exceed_axis1_vx*/

static const char log_softmax_exceed_axis1_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         depth;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_EXCEED_AXIS1_BF16(read_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    for (coord.z = 0; coord.z < depth; coord.z ++) \\\n\
    { \\\n\
        for (coord.y = 0; coord.y < axisSize;) \\\n\
        { \\\n\
            read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            _viv_asm(COPY, vec0, in0, 16); \\\n\
            VXC_VertMax3_Half(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
        } \\\n\
    } \\\n\
    _viv_asm(COPY, tmp0, max, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, max_lo, tmp1, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, max_hi, tmp1, 16); \\\n\
    coord.y = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    for (coord.z = 0; coord.z < depth; coord.z ++) \\\n\
    { \\\n\
        for (coord.y = 0; coord.y < axisSize;) \\\n\
        { \\\n\
            read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                                                             uniConvBF16toF32_Part0_2x8); \\\n\
            _viv_asm(COPY, data0, tmp1, 16); \\\n\
            data0 = data0 - max_lo; \\\n\
            data0 *= scaleLogE; \\\n\
            sum0  += exp2(data0); \\\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                                                              uniConvBF16toF32_Part1_2x8); \\\n\
            _viv_asm(COPY, data0, tmp1, 16); \\\n\
            data0 = data0 - max_hi; \\\n\
            data0 *= scaleLogE; \\\n\
            sum1  += exp2(data0); \\\n\
            coord.y++; \\\n\
        } \\\n\
    } \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
__kernel void log_softmax_exceed_axis1_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    for (coord.z = 0; coord.z < depth; coord.z ++)\n\
    {\n\
        for (coord.y = 0; coord.y < axisSize;)\n\
        {\n\
            VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
            _viv_asm(COPY, data0, tmp1, 16);\n\
            data0 = data0 - max_lo;\n\
            data0 = data0 * betaValue - sum0;\n\
            _viv_asm(COPY, dst0, data0, 16);\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, data0, tmp1, 16);\n\
            data0 = data0 - max_hi;\n\
            data0 = data0 * betaValue - sum1;\n\
            _viv_asm(COPY, dst1, data0, 16);\n\
            VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
            VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_exceed_axis1_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    half4 dst0, dst1;\n\
    for (coord.z = 0; coord.z < depth; coord.z ++)\n\
    {\n\
        for (coord.y = 0; coord.y < axisSize;)\n\
        {\n\
            VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
            _viv_asm(COPY, data0, tmp1, 16);\n\
            data0 = data0 - max_lo;\n\
            data0 = data0 * betaValue - sum0;\n\
            _viv_asm(CONV, dst0, data0);\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, data0, tmp1, 16);\n\
            data0 = data0 - max_hi;\n\
            data0 = data0 * betaValue - sum1;\n\
            _viv_asm(CONV, dst1, data0);\n\
            VXC_DP2x8(vec0, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
            vxc_short8 vect;\n\
            _viv_asm(COPY, vect, vec0, 16);\n\
            VXC_WriteImage2DArray(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_exceed_axis1_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_EXCEED_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    for (coord.z = 0; coord.z < depth; coord.z ++)\n\
    {\n\
        for (coord.y = 0; coord.y < axisSize;)\n\
        {\n\
            VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
            _viv_asm(COPY, data0, tmp1, 16);\n\
            data0 = data0 - max_lo;\n\
            data0 = data0 * betaValue - sum0;\n\
            write_imagef(output, coord, data0);\n\
            coord.x += 4;\n\
            VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, data0, tmp1, 16);\n\
            data0 = data0 - max_hi;\n\
            data0 = data0 * betaValue - sum1;\n\
            write_imagef(output, coord, data0);\n\
            coord.x -= 4;\n\
            coord.y++;\n\
        }\n\
    }\n\
}\n\
"; /* end of log_softmax_exceed_axis1_BF16_vx*/

static const char logical_not_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void logical_not_I8toI8(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_char8 src0;\n\
    vxc_char8 dst;\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    dst = !src0;\n\
    dst *= (-1);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void logical_not_I8toI8_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char8 src0;\n\
    vxc_char8 dst;\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    dst = !src0;\n\
    dst *= (-1);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of logical_not_vx*/

static const char logical_ops_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16toInt8_2x8;\n\
\n\
#define TENSORLOGICAL_PROCESS(input_type, copy_type, output_type, out_copy_type,\\\n\
lgc_op, lgc_op2, read_fun, write_fun) \\\n\
    input_type vA;\\\n\
    copy_type  src0;\\\n\
    input_type vB;\\\n\
    copy_type  src1;\\\n\
    read_fun(vA,in0,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src0, vA, 16); \\\n\
    read_fun(vB,in1,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src1, vB, 16); \\\n\
    output_type dst; \\\n\
    dst = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    dst *= (-1); \\\n\
    out_copy_type data; \\\n\
    _viv_asm(COPY, data, dst, 16); \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
\n\
#define TENSORLOGICAL(name0, src_type_name, dst_type_name, input_type, copy_type,\\\n\
output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\\\n\
    TENSORLOGICAL_PROCESS(input_type, copy_type, output_type, out_copy_type,\\\n\
    lgc_op, lgc_op2, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
#define TENSORLOGICAL_2D(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    TENSORLOGICAL_PROCESS(input_type, copy_type, output_type, out_copy_type,\\\n\
    lgc_op, lgc_op2, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniMulShortMinus1toFp16_2x8;\n\
\n\
#define TENSORLOGICAL_FP_PROCESS(input_type, copy_type, output_type,\\\n\
out_copy_type, lgc_op, lgc_op2,  read_fun, write_fun) \\\n\
    input_type vA;\\\n\
    copy_type  src0;\\\n\
    input_type vB;\\\n\
    copy_type  src1;\\\n\
    read_fun(vA,in0,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src0, vA, 16); \\\n\
    read_fun(vB,in1,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src1, vB, 16); \\\n\
    output_type dst; \\\n\
    dst = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    vxc_half8 tmpOut; \\\n\
    VXC_DP2x8(tmpOut,dst,dst,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero, 0),uniMulShortMinus1toFp16_2x8); \\\n\
    out_copy_type data; \\\n\
    _viv_asm(COPY, data, tmpOut, 16); \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define TENSORLOGICAL_FP(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\\\n\
    TENSORLOGICAL_FP_PROCESS(input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2,\\\n\
    VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
#define TENSORLOGICAL_FP_2D(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    TENSORLOGICAL_FP_PROCESS(input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2,\\\n\
    VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
#define TENSORLOGICAL_BFP_PROCESS(input_type, copy_type, output_type,\\\n\
out_copy_type, lgc_op, lgc_op2,  read_fun, write_fun) \\\n\
    input_type vA;\\\n\
    copy_type  src0;\\\n\
    input_type vB;\\\n\
    copy_type  src1;\\\n\
    read_fun(vA,in0,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src0, vA, 16); \\\n\
    read_fun(vB,in1,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src1, vB, 16); \\\n\
    output_type dst; \\\n\
    dst = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    vxc_char8 data; \\\n\
    VXC_DP2x8(data,dst,dst,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero, 1),uniConvertInt16toInt8_2x8); \\\n\
    data &= 1; \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define TENSORLOGICAL_BFP16(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\\\n\
    TENSORLOGICAL_BFP_PROCESS(input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2,\\\n\
    VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
#define TENSORLOGICAL_BFP16_2D(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    TENSORLOGICAL_BFP_PROCESS(input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2,\\\n\
    VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
//          name0, src_name, dst_name, input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2\n\
TENSORLOGICAL(or,      I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ||, )\n\
//TENSORLOGICAL(or,      U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ||, )\n\
//TENSORLOGICAL(or,      I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
//TENSORLOGICAL_FP(or,   F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
TENSORLOGICAL(and,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   &&, )\n\
//TENSORLOGICAL(and,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  &&, )\n\
//TENSORLOGICAL(and,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
//TENSORLOGICAL_FP(and,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
TENSORLOGICAL(xor,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ^, !!)\n\
//TENSORLOGICAL(xor,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ^, !!)\n\
//TENSORLOGICAL(xor,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
//TENSORLOGICAL_FP(xor,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
\n\
TENSORLOGICAL_BFP16(or,   BF16,   I8,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
TENSORLOGICAL_BFP16(and,  BF16,   I8,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
TENSORLOGICAL_BFP16(xor,  BF16,   I8,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
\n\
TENSORLOGICAL_2D(or,      I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ||, )\n\
//TENSORLOGICAL_2D(or,      U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ||, )\n\
//TENSORLOGICAL_2D(or,      I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
//TENSORLOGICAL_FP_2D(or,   F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
TENSORLOGICAL_2D(and,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   &&, )\n\
//TENSORLOGICAL_2D(and,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  &&, )\n\
//TENSORLOGICAL_2D(and,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
//TENSORLOGICAL_FP_2D(and,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
TENSORLOGICAL_2D(xor,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ^, !!)\n\
//TENSORLOGICAL_2D(xor,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ^, !!)\n\
//TENSORLOGICAL_2D(xor,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
//TENSORLOGICAL_FP_2D(xor,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
\n\
TENSORLOGICAL_BFP16_2D(or,   BF16,  I8,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
TENSORLOGICAL_BFP16_2D(and,  BF16,  I8,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
TENSORLOGICAL_BFP16_2D(xor,  BF16,  I8,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
"; /* end of logical_ops_vx*/

static const char lstmunit_activation_BP_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_BP_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_DP2x8(data1, vect10, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src10, data1, 16); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data0, 16); \\\n\
    VXC_DP2x8(data1, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
    VXC_DP2x8(data1, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
    data_i_t = src0 + src10 + b0; \\\n\
    data_f_t = src1 + src11 + b1; \\\n\
    data_g_t = src2 + src12 + b2; \\\n\
    data_o_t = src3 + src13 + b3; \\\n\
 \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_BP_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_BF16_vx*/

static const char lstmunit_activation_BP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_BP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_BP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_F16_vx*/

static const char lstmunit_activation_BP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_BP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_BP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_U8_vx*/

static const char lstmunit_activation_B_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_B_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_DP2x8(data1, vect10, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src10, data1, 16); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data0, 16); \\\n\
    VXC_DP2x8(data1, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
    VXC_DP2x8(data1, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
    data_i_t = src0 + src10 + b0; \\\n\
    data_f_t = src1 + src11 + b1; \\\n\
    data_g_t = src2 + src12 + b2; \\\n\
    data_o_t = src3 + src13 + b3; \\\n\
 \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_B_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_B_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_B_BF16_vx*/

static const char lstmunit_activation_B_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_B_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_B_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_B_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_B_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_B_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_B_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_B_F16_vx*/

static const char lstmunit_activation_B_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_B_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
LSTMUNIT_B_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_B_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_B_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_B_U8_vx*/

static const char lstmunit_activation_CBP_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_CBP_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data0, 16); \\\n\
    VXC_DP2x8(data1, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
    VXC_DP2x8(data1, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
    data_f_t = src1 + src11 + b1; \\\n\
    data_g_t = src2 + src12 + b2; \\\n\
    data_o_t = src3 + src13 + b3; \\\n\
 \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_CBP_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_BF16_vx*/

static const char lstmunit_activation_CBP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CBP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CBP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_F16_vx*/

static const char lstmunit_activation_CBP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CBP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CBP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_U8_vx*/

static const char lstmunit_activation_CB_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_CB_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data0, 16); \\\n\
    VXC_DP2x8(data1, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
    VXC_DP2x8(data1, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
    data_f_t = src1 + src11 + b1; \\\n\
    data_g_t = src2 + src12 + b2; \\\n\
    data_o_t = src3 + src13 + b3; \\\n\
 \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_CB_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_BF16_vx*/

static const char lstmunit_activation_CB_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CB_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CB_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_F16_vx*/

static const char lstmunit_activation_CB_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CB_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CB_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_U8_vx*/

static const char lstmunit_activation_CLP_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
#define LSTMUNIT_CLP_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_f_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_g_t, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_o_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data1, 16); \\\n\
 \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CLP_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_CLP_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CLP_BF16_vx*/

static const char lstmunit_activation_CLP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_CLP_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CLP_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CLP_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CLP_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CLP_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
#define LSTMUNIT_CLP_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CLP_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CLP_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CLP_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CLP_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CLP_F16_vx*/

static const char lstmunit_activation_CL_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_CL_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_f_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_g_t, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_o_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data1, 16); \\\n\
 \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CL_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_CL_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CL_BF16_vx*/

static const char lstmunit_activation_CL_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
#define LSTMUNIT_CL_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CL_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CL_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CL_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CL_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CL_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CL_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CL_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CL_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CL_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CL_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CL_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CL_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CL_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CL_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CL_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CL_F16_vx*/

static const char lstmunit_activation_CSP_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_CSP_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data0, 16); \\\n\
    VXC_DP2x8(data1, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
    VXC_DP2x8(data1, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
 \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_CSP_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_BF16_vx*/

static const char lstmunit_activation_CSP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CSP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CSP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_F16_vx*/

static const char lstmunit_activation_CSP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CSP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CSP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_U8_vx*/

static const char lstmunit_activation_CS_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_CS_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data0, 16); \\\n\
    VXC_DP2x8(data1, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
    VXC_DP2x8(data1, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
 \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_CS_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CS_BF16_vx*/

static const char lstmunit_activation_CS_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CS_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CS_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CS_F16_vx*/

static const char lstmunit_activation_CS_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CS_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CS_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CS_U8_vx*/

static const char lstmunit_activation_LP_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_LP_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_LP_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_i_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_f_t, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_g_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_o_t, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
 \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_LP_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_LP_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_LP_BF16_vx*/

static const char lstmunit_activation_LP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
#define LSTMUNIT_LP_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_LP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_LP_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_LP_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_LP_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_LP_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_LP_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_LP_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_LP_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_LP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_LP_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_LP_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_LP_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_LP_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_LP_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_LP_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_LP_F16_vx*/

static const char lstmunit_activation_L_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_L_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_L_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(data0, vect0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_i_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_f_t, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_g_t, data0, 16); \\\n\
    VXC_DP2x8(data1, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_o_t, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
 \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_L_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_L_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_L_BF16_vx*/

static const char lstmunit_activation_L_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
#define LSTMUNIT_L_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_L_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_L_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_L_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_L_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_L_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_L_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_L_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_L_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_L_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_L_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_L_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_L_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_L_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_L_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_L_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
"; /* end of lstmunit_activation_L_F16_vx*/

static const char lstmunit_activation_SP_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_SP_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1, data2, data3; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_float4  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(data0, vect0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_DP2x8(data1, vect10, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src10, data1, 16); \\\n\
    VXC_DP2x8(data2, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data2, 16); \\\n\
    VXC_DP2x8(data3, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data3, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data2, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data2, 16); \\\n\
    VXC_DP2x8(data3, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data3, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
 \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, data0, data_c_t, 16); \\\n\
    VXC_DP2x8(data1, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, data0, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_SP_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_BF16_vx*/

static const char lstmunit_activation_SP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_SP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_SP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_F16_vx*/

static const char lstmunit_activation_SP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_SP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_U8_FP32(U8,  SIGMOID, int4, vxc_uchar4,  vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_SP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_U8_FP16(U8,  SIGMOID, int4, vxc_uchar4,  vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_U8_vx*/

static const char lstmunit_activation_S_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_S_BF16(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_BF16toBF16_BF16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4, data0, data1; \\\n\
    vxc_float4 src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_float4 src10, src11, src12, src13; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(data0, vect0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_DP2x8(data1, vect10, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src10, data1, 16); \\\n\
    VXC_DP2x8(data0, vect1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src1, data0, 16); \\\n\
    VXC_DP2x8(data1, vect11, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src11, data1, 16); \\\n\
    VXC_DP2x8(data0, vect2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src2, data0, 16); \\\n\
    VXC_DP2x8(data1, vect12, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src12, data1, 16); \\\n\
    VXC_DP2x8(data0, vect3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src3, data0, 16); \\\n\
    VXC_DP2x8(data1, vect13, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, src13, data1, 16); \\\n\
    VXC_DP2x8(data0, vect4, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data_c_t, data0, 16); \\\n\
 \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
 \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(COPY, vect0, data_c_t, 16); \\\n\
    VXC_DP2x8(data0, vect0, vect0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(COPY, vect1, data_o_t, 16); \\\n\
    VXC_DP2x8(data1, vect1, vect1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_BF16(SIGMOID, sigmoid)\n\
LSTMUNIT_S_BF16(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_BF16_vx*/

static const char lstmunit_activation_S_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_S_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_S_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_S_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_S_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_F16_vx*/

static const char lstmunit_activation_S_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_S_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_S_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_U8_vx*/

static const char matrixmul_bf16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void gemm_BF16BF16toBF16(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB,\n\
            int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_ushort8 valC0, valC1, src0, src1;\n\
    vxc_ushort8 srcA0, srcB0, srcA1, srcB1, outC;\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 4;\n\
        coord_b.y += 4;\n\
\n\
        VXC_DP2x8(src0, srcA0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempA0, src0, 16);\n\
        VXC_DP2x8(src1, srcA0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, tempA1, src1, 16);\n\
        VXC_DP2x8(src0, srcA1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempA2, src0, 16);\n\
        VXC_DP2x8(src1, srcA1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, tempA3, src1, 16);\n\
        VXC_DP2x8(src0, srcB0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempB0, src0, 16);\n\
        VXC_DP2x8(src1, srcB0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, tempB1, src1, 16);\n\
        VXC_DP2x8(src0, srcB1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempB2, src0, 16);\n\
        VXC_DP2x8(src1, srcB1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, tempB3, src1, 16);\n\
\n\
        sum0 += (tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3);\n\
        sum1 += (tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3);\n\
        sum2 += (tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3);\n\
        sum3 += (tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3);\n\
    }\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(COPY, valC0, sum0, 16);\n\
    _viv_asm(COPY, valC1, sum1, 16);\n\
    VXC_DP2x8(outC, valC0, valC1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(COPY, valC0, sum2, 16);\n\
    _viv_asm(COPY, valC1, sum3, 16);\n\
    VXC_DP2x8(outC, valC0, valC1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gemm_transa_BF16BF16toBF16(\n\
                        image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
\n\
    vxc_ushort8 valC0, valC1;\n\
    vxc_ushort8 srcA, srcB, outC, src0, src1;\n\
\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    vxc_float4 tempA0;\n\
    vxc_float4 tempB0;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    for(coord_a.y = 0, coord_b.y = 0; coord_a.y < K;)\n\
    {\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.y++;\n\
        coord_b.y++;\n\
\n\
        VXC_DP2x8(src0, srcA, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempA0, src0, 16);\n\
\n\
        VXC_DP2x8(src1, srcB, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempB0, src1, 16);\n\
\n\
        sum0 = (sum0 + tempA0.x * tempB0);\n\
        sum1 = (sum1 + tempA0.y * tempB0);\n\
        sum2 = (sum2 + tempA0.z * tempB0);\n\
        sum3 = (sum3 + tempA0.w * tempB0);\n\
    }\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(COPY, valC0, sum0, 16);\n\
    _viv_asm(COPY, valC1, sum1, 16);\n\
    VXC_DP2x8(outC, valC0, valC1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    _viv_asm(COPY, valC0, sum2, 16);\n\
    _viv_asm(COPY, valC1, sum3, 16);\n\
    VXC_DP2x8(outC, valC0, valC1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gemm_transb_BF16BF16toBF16(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_ushort8 src0, src1;\n\
\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_ushort8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_ushort8 srcB0,srcB1,srcB2,srcB3;\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 4;\n\
        coord_b.x += 4;\n\
\n\
        VXC_DP2x8(src0, srcA0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempA0, src0, 16);\n\
        VXC_DP2x8(src1, srcA1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempA1, src1, 16);\n\
        VXC_DP2x8(src0, srcA2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempA2, src0, 16);\n\
        VXC_DP2x8(src1, srcA3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempA3, src1, 16);\n\
\n\
        VXC_DP2x8(src0, srcB0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempB0, src0, 16);\n\
        VXC_DP2x8(src1, srcB1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempB1, src1, 16);\n\
        VXC_DP2x8(src0, srcB2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempB2, src0, 16);\n\
        VXC_DP2x8(src1, srcB3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, tempB3, src1, 16);\n\
\n\
        sum0 += (float4)(dot(tempA0, tempB0), dot(tempA0, tempB1), dot(tempA0, tempB2), dot(tempA0, tempB3));\n\
        sum1 += (float4)(dot(tempA1, tempB0), dot(tempA1, tempB1), dot(tempA1, tempB2), dot(tempA1, tempB3));\n\
        sum2 += (float4)(dot(tempA2, tempB0), dot(tempA2, tempB1), dot(tempA2, tempB2), dot(tempA2, tempB3));\n\
        sum3 += (float4)(dot(tempA3, tempB0), dot(tempA3, tempB1), dot(tempA3, tempB2), dot(tempA3, tempB3));\n\
    }\n\
\n\
    vxc_ushort8 valC0, valC1, valDst;\n\
    _viv_asm(COPY, valC0, sum0, 16);\n\
    _viv_asm(COPY, valC1, sum1, 16);\n\
    VXC_DP2x8(valDst, valC0, valC1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    _viv_asm(COPY, valC0, sum2, 16);\n\
    _viv_asm(COPY, valC1, sum3, 16);\n\
    VXC_DP2x8(valDst, valC0, valC1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of matrixmul_bf16_vx*/

static const char matrixmul_cross_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float mulKIn0In1Zp;\n\
_viv_uniform float inOutScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8U8toFp32Block4_4x4;\n\
_viv_uniform VXC_512Bits uniGemmU8U8MulZptoFp32_8x4;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
\n\
#define GEMM_QINT_TO_QINT_CROSS(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name##_cross( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N, \\\n\
        int axis_size, int inner_size, int outer_size, int axis_size0, \\\n\
        int inner_size0, int outer_size0, int axis_size1, int inner_size1, \\\n\
        int outer_size1, int axis_size2, int inner_size2, int outer_size2) \\\n\
{ \\\n\
    read_type srcA0, srcA1, srcA2, srcA3, srcB, outC; \\\n\
    vxc_float4 sum = (vxc_float4)(mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp); \\\n\
    int gidz = get_global_id(2); \\\n\
    for(int j = 0; j < outer_size; j++) \\\n\
    { \\\n\
        for(int i = 0; i < inner_size; i++) \\\n\
        { \\\n\
            vxc_float4 sum0 = sum, sum1 = sum, sum2 = sum, sum3 = sum; \\\n\
            int4 coord_a = (int4)(0, get_global_id(1), gidz * axis_size0 + i * inner_size0 + j * outer_size0, 0); \\\n\
            int4 coord_b = (int4)(get_global_id(0), 0, gidz * axis_size1 + i * inner_size1 + j * outer_size1, 0); \\\n\
            int8 inputA_desc, inputB_desc, output_desc; \\\n\
            _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
            int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
            _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
            _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
            int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
            _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
            for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
            { \\\n\
                vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
                vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
                VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                            VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                            VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                            VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
                coord_a.x += 4; coord_b.y += 4; \\\n\
                VXC_DP4x4(tempA0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8toFp32Block4_4x4); \\\n\
                VXC_DP4x4(tempA1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8toFp32Block4_4x4); \\\n\
                VXC_DP4x4(tempA2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8toFp32Block4_4x4); \\\n\
                VXC_DP4x4(tempA3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8toFp32Block4_4x4); \\\n\
                VXC_DP8x4(tempB0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8MulZptoFp32_8x4); \\\n\
                VXC_DP8x4(tempB1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8MulZptoFp32_8x4); \\\n\
                VXC_DP8x4(tempB2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8MulZptoFp32_8x4); \\\n\
                VXC_DP8x4(tempB3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniGemmU8U8MulZptoFp32_8x4); \\\n\
                sum0 += tempA0 + tempB0; \\\n\
                sum1 += tempA1 + tempB1; \\\n\
                sum2 += tempA2 + tempB2; \\\n\
                sum3 += tempA3 + tempB3; \\\n\
            } \\\n\
            vxc_int4 tmpOut0, tmpOut1; \\\n\
            coord_b.y = get_global_id(1); \\\n\
            coord_b.z = gidz * axis_size2 + i * inner_size2 + j * outer_size2; \\\n\
            _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
            int baseAddr = (int)coord_b.z * output_desc.s4 + output_desc.s0; \\\n\
            _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
            tmpOut0 = convert_int4_rte(sum0 * inOutScale + output_ZP); \\\n\
            tmpOut1 = convert_int4_rte(sum1 * inOutScale + output_ZP); \\\n\
            VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
            coord_b.y++; \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
            coord_b.y++; \\\n\
            tmpOut0 = convert_int4_rte(sum2 * inOutScale + output_ZP); \\\n\
            tmpOut1 = convert_int4_rte(sum3 * inOutScale + output_ZP); \\\n\
            VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
            coord_b.y++; \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
}\n\
GEMM_QINT_TO_QINT_CROSS(U8, vxc_uchar16)\n\
GEMM_QINT_TO_QINT_CROSS(I8, vxc_char16)\n\
\n\
__kernel void gemm_F16F16toF16_cross(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N,\n\
            int axis_size, int inner_size, int outer_size, int axis_size0,\n\
            int inner_size0, int outer_size0, int axis_size1, int inner_size1,\n\
            int outer_size1, int axis_size2, int inner_size2, int outer_size2)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    uint gidz = get_global_id(2);\n\
    for(int j = 0; j < outer_size; j++)\n\
    {\n\
        for(int i = 0; i < inner_size; i++)\n\
        {\n\
            int4 coord_a = (int4)(0, gidy, gidz * axis_size0 + i * inner_size0 + j * outer_size0, 0);\n\
            int4 coord_b = (int4)(get_global_id(0), 0, gidz * axis_size1 + i * inner_size1 + j * outer_size1, 0);\n\
\n\
            half4 valC;\n\
            vxc_short8 srcA0, srcA1, srcA2, srcA3, outC;\n\
            vxc_half8 tmpA0, tmpA1, tmpA2, tmpA3;\n\
            vxc_short16 srcB;\n\
            vxc_half16 tmpB;\n\
            vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
            vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
            int8 inputA_desc, inputB_desc, output_desc;\n\
            _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
            int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
            _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
            _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
            int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
            _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
            for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
            {\n\
                vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
                VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
                VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
                VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
                VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                            VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
                VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
                VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
                VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
                VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                            VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
                coord_a.x += 4; coord_b.y += 4;\n\
                _viv_asm(COPY, tmpA0, srcA0, 16);\n\
                _viv_asm(COPY, tmpA1, srcA1, 16);\n\
                _viv_asm(COPY, tmpA2, srcA2, 16);\n\
                _viv_asm(COPY, tmpA3, srcA3, 16);\n\
                _viv_asm(COPY, tmpB.hi, srcB.hi, 16);\n\
                _viv_asm(COPY, tmpB.lo, srcB.lo, 16);\n\
                VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                            uniGemmU8F16toF32Lo_4x4b);\n\
                VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                            uniGemmU8F16toF32Lo_4x4b);\n\
                VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, tmpA2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                            uniGemmU8F16toF32Lo_4x4b);\n\
                VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, tmpA3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                            uniGemmU8F16toF32Lo_4x4b);\n\
                sum0 += (tempA0);\n\
                sum1 += (tempA1);\n\
                sum2 += (tempA2);\n\
                sum3 += (tempA3);\n\
            }\n\
            coord_b.y = gidy;\n\
            coord_b.z = gidz * axis_size2 + i * inner_size2 + j * outer_size2;\n\
            _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
            int baseAddr = (int)coord_b.z * output_desc.s4 + output_desc.s0;\n\
            _viv_asm(MOV, coord_b.w, baseAddr);\n\
            _viv_asm(CONV, valC, sum0);\n\
            _viv_asm(COPY, outC, valC, 16);\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246,\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
            coord_b.y++;\n\
            _viv_asm(CONV, valC, sum1);\n\
            _viv_asm(COPY, outC, valC, 16);\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246,\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
            coord_b.y++;\n\
            _viv_asm(CONV, valC, sum2);\n\
            _viv_asm(COPY, outC, valC, 16);\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246,\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
            coord_b.y++;\n\
            _viv_asm(CONV, valC, sum3);\n\
            _viv_asm(COPY, outC, valC, 16);\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246,\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
}\n\
"; /* end of matrixmul_cross_vx*/

static const char matrixmul_cross_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform int outer;\n\
\n\
#define GEMM_QINT_TO_QINT_MERGE(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name##_merge( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
    for(int i = 0; i < outer; i++) \\\n\
    { \\\n\
        read_type srcA, srcB, outC; \\\n\
        int4 coord_a = (int4)(0, gidy, (ac2zero ? i : get_global_id(2)), 0); \\\n\
        int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? i : get_global_id(2)), 0); \\\n\
        vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
        vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
     \\\n\
        int8 inputA_desc, inputB_desc, output_desc; \\\n\
        _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
        int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
        _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
        _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
        int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
        _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
     \\\n\
        for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
        { \\\n\
            vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
            vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
            VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32_4x4); \\\n\
            VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32B_4x4); \\\n\
            VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32_4x4); \\\n\
            VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32B_4x4); \\\n\
            VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32_4x4); \\\n\
            VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32B_4x4); \\\n\
            VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord_a.x += 4; \\\n\
            coord_b.y += 4; \\\n\
            VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32_4x4); \\\n\
            VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvertUint8SubZpToFp32B_4x4); \\\n\
            sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
            sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
            sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
            sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
        } \\\n\
        vxc_int4 tmpOut0, tmpOut1; \\\n\
        coord_b.y = gidy; \\\n\
        coord_b.z = get_global_id(2) + i * get_global_size(2); \\\n\
        _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
        int baseAddr = (int)coord_b.z * output_desc.s4 + output_desc.s0; \\\n\
        _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
        tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
        tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
        VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_b.y++; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_b.y++; \\\n\
        tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
        tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
        VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_b.y++; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
GEMM_QINT_TO_QINT_MERGE(I16, vxc_short8)\n\
\n\
#define GEMM_QINT_TO_QINT_CROSS(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name##_cross( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N, \\\n\
        int axis_size, int inner_size, int outer_size, int axis_size0, \\\n\
        int inner_size0, int outer_size0, int axis_size1, int inner_size1, \\\n\
        int outer_size1, int axis_size2, int inner_size2, int outer_size2) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
    for(int j = 0; j < outer_size; j++) \\\n\
    { \\\n\
        for(int i = 0; i < inner_size; i++) \\\n\
        { \\\n\
            read_type srcA, srcB, outC; \\\n\
            int4 coord_a = (int4)(0, gidy, gidz * axis_size0 + i * inner_size0 + j * outer_size0, 0); \\\n\
            int4 coord_b = (int4)(get_global_id(0), 0, gidz * axis_size1 + i * inner_size1 + j * outer_size1, 0); \\\n\
            vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
            vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
         \\\n\
            int8 inputA_desc, inputB_desc, output_desc; \\\n\
            _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
            int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
            _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
            _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
            int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
            _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
         \\\n\
            for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
            { \\\n\
                vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
                vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
                VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32_4x4); \\\n\
                VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32B_4x4); \\\n\
                VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32_4x4); \\\n\
                VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32B_4x4); \\\n\
                VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32_4x4); \\\n\
                VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32B_4x4); \\\n\
                VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
                coord_a.x += 4; \\\n\
                coord_b.y += 4; \\\n\
                VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32_4x4); \\\n\
                VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                            uniConvertUint8SubZpToFp32B_4x4); \\\n\
                sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
                sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
                sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
                sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
            } \\\n\
            vxc_int4 tmpOut0, tmpOut1; \\\n\
            coord_b.y = gidy; \\\n\
            coord_b.z = gidz * axis_size2 + i * inner_size2 + j * outer_size2; \\\n\
            _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
            int baseAddr = (int)coord_b.z * output_desc.s4 + output_desc.s0; \\\n\
            _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
            tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
            tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
            VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
            coord_b.y++; \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
            coord_b.y++; \\\n\
            tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
            tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
            VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                        uniConvertInt32toUint8_2x8); \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
            coord_b.y++; \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
}\n\
GEMM_QINT_TO_QINT_CROSS(I16, vxc_short8)\n\
"; /* end of matrixmul_cross_i16_vx*/

static const char matrixmul_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Hi_4x4;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#if (VX_VERSION==2)\n\
__kernel void gemm_F16F16toF16(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB,\n\
            int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    half4 valC;\n\
    vxc_short8 srcA0, srcA1, srcA2, srcA3, outC;\n\
    vxc_half8 tmpA0, tmpA1, tmpA2, tmpA3;\n\
    vxc_short16 srcB;\n\
    vxc_half16 tmpB;\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 4;\n\
        coord_b.y += 4;\n\
        _viv_asm(COPY, tmpA0, srcA0, 16);\n\
        _viv_asm(COPY, tmpA1, srcA1, 16);\n\
        _viv_asm(COPY, tmpA2, srcA2, 16);\n\
        _viv_asm(COPY, tmpA3, srcA3, 16);\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16);\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16);\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, tmpA2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, tmpA3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        sum0 += (tempA0);\n\
        sum1 += (tempA1);\n\
        sum2 += (tempA2);\n\
        sum3 += (tempA3);\n\
    }\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(CONV, valC, sum0);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum1);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum2);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum3);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
#else\n\
__kernel void gemm_F16F16toF16(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB,\n\
            int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    half4 valC;\n\
    vxc_short8 srcA0, srcB0, srcA1, srcB1, outC;\n\
    vxc_half8 tmpA0, tmpB0, tmpA1, tmpB1;\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 4;\n\
        coord_b.y += 4;\n\
        _viv_asm(COPY, tmpA0, srcA0, 16);\n\
        _viv_asm(COPY, tmpB0, srcB0, 16);\n\
        _viv_asm(COPY, tmpA1, srcA1, 16);\n\
        _viv_asm(COPY, tmpB1, srcB1, 16);\n\
\n\
        VXC_DP4x4(tempA0, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4);\n\
        VXC_DP4x4(tempB0, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4);\n\
        VXC_DP4x4(tempA1, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4);\n\
        VXC_DP4x4(tempB1, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4);\n\
        VXC_DP4x4(tempA2, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4);\n\
        VXC_DP4x4(tempB2, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4);\n\
        VXC_DP4x4(tempA3, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4);\n\
        VXC_DP4x4(tempB3, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4);\n\
        sum0 += (tempA0 + tempB0);\n\
        sum1 += (tempA1 + tempB1);\n\
        sum2 += (tempA2 + tempB2);\n\
        sum3 += (tempA3 + tempB3);\n\
    }\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(CONV, valC, sum0);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum1);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum2);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum3);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
#endif\n\
\n\
__kernel void gemm_F32F32toF32(\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output,\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(gidx, 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    vxc_uchar16 outC;\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8);\n\
\n\
    for(int i = 0; i < K; i+=4)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
\n\
        coord_a.x = i;\n\
        coord_a.y = gidy;\n\
        coord_b.x = gidx;\n\
        coord_b.y = i;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        coord_a.y++;\n\
        tempA1 = read_imagef(inputA, coord_a);\n\
        coord_a.y++;\n\
        tempA2 = read_imagef(inputA, coord_a);\n\
        coord_a.y++;\n\
        tempA3 = read_imagef(inputA, coord_a);\n\
\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_b.y++;\n\
        tempB1 = read_imagef(inputB, coord_b);\n\
        coord_b.y++;\n\
        tempB2 = read_imagef(inputB, coord_b);\n\
        coord_b.y++;\n\
        tempB3 = read_imagef(inputB, coord_b);\n\
\n\
        sum0 += (tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3);\n\
        sum1 += (tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3);\n\
        sum2 += (tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3);\n\
        sum3 += (tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3);\n\
    }\n\
    coord_b = (int4)(gidx, gidy, get_global_id(2), get_global_id(2));\n\
    write_imagef(output, coord_b, sum0);\n\
    coord_b.y++;\n\
    write_imagef(output, coord_b, sum1);\n\
    coord_b.y++;\n\
    write_imagef(output, coord_b, sum2);\n\
    coord_b.y++;\n\
    write_imagef(output, coord_b, sum3);\n\
}\n\
"; /* end of matrixmul_f16_vx*/

static const char matrixmul_f16f16_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Hi_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_F16_TO_QINT(dst_type_name, write_type) \\\n\
__kernel void gemm_F16F16to##dst_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1, srcA2, srcA3; \\\n\
    vxc_half8 tmpA0, tmpA1, tmpA2, tmpA3; \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        _viv_asm(COPY, tmpA2, srcA2, 16); \\\n\
        _viv_asm(COPY, tmpA3, srcA3, 16); \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, tmpA2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, tmpA3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        sum0 += (tempA0); \\\n\
        sum1 += (tempA1); \\\n\
        sum2 += (tempA2); \\\n\
        sum3 += (tempA3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    write_type outC; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#else\n\
#define GEMM_F16_TO_QINT(dst_type_name, write_type) \\\n\
__kernel void gemm_F16F16to##dst_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcB0, srcA1, srcB1; \\\n\
    vxc_half8 tmpA0, tmpB0, tmpA1, tmpB1; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpB0, srcB0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        _viv_asm(COPY, tmpB1, srcB1, 16); \\\n\
 \\\n\
        VXC_DP4x4(tempA0, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4); \\\n\
        VXC_DP4x4(tempB0, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4); \\\n\
        VXC_DP4x4(tempB1, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4); \\\n\
        VXC_DP4x4(tempB2, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4); \\\n\
        VXC_DP4x4(tempB3, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4); \\\n\
        sum0 += (tempA0 + tempB0); \\\n\
        sum1 += (tempA1 + tempB1); \\\n\
        sum2 += (tempA2 + tempB2); \\\n\
        sum3 += (tempA3 + tempB3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    write_type outC; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#endif\n\
GEMM_F16_TO_QINT(U8, vxc_uchar16)\n\
GEMM_F16_TO_QINT(I8, vxc_char16)\n\
GEMM_F16_TO_QINT(I16, vxc_short8)\n\
"; /* end of matrixmul_f16f16_u8_vx*/

static const char matrixmul_f16i16_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32A_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32B_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32C_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32D_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32Lo_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32Hi_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmFp16I16MulZptoFp32_4x4;\n\
_viv_uniform float in1outScale;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_F16_QINT16_TO_QINT16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toI16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1, outC; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    vxc_short16 srcB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4_b(tempA0, srcB.hi, srcB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, srcB.hi, srcB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Hi_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, srcB.hi, srcB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, srcB.hi, srcB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Hi_4x4b); \\\n\
        VXC_DP4x4(tmpZpScale, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16I16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tmpZpScale.xxxx; \\\n\
        sum1 += tempA1 + tmpZpScale.yyyy; \\\n\
        sum2 += tempA2 + tmpZpScale.zzzz; \\\n\
        sum3 += tempA3 + tmpZpScale.wwww; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#else\n\
#define GEMM_F16_QINT16_TO_QINT16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toI16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1, outC; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    vxc_short8 srcB0, srcB1; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA0, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32A_4x4); \\\n\
        VXC_DP4x4(tempB0, tmpA0, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32B_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32C_4x4); \\\n\
        VXC_DP4x4(tempB1, tmpA0, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32D_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32A_4x4); \\\n\
        VXC_DP4x4(tempB2, tmpA1, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32B_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32C_4x4); \\\n\
        VXC_DP4x4(tempB3, tmpA1, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32D_4x4); \\\n\
        VXC_DP4x4(tmpZpScale, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16I16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempB0 + tmpZpScale.xxxx; \\\n\
        sum1 += tempA1 + tempB1 + tmpZpScale.yyyy; \\\n\
        sum2 += tempA2 + tempB2 + tmpZpScale.zzzz; \\\n\
        sum3 += tempA3 + tempB3 + tmpZpScale.wwww; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#endif\n\
GEMM_F16_QINT16_TO_QINT16(I16, vxc_short8)\n\
\n\
"; /* end of matrixmul_f16i16_i16_vx*/

static const char matrixmul_f16u8_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16U8MulZptoFp32_4x4;\n\
_viv_uniform float input1Scale;\n\
\n\
#define GEMM_F16_QINT_TO_F16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toF16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    half4 valC; \\\n\
    vxc_short8 srcA0, srcA1, outC; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    read_type srcB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempZp; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempZp, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16U8MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempZp.x; \\\n\
        sum1 += tempA1 + tempZp.y; \\\n\
        sum2 += tempA2 + tempZp.z; \\\n\
        sum3 += tempA3 + tempZp.w; \\\n\
    } \\\n\
    sum0 *= input1Scale; \\\n\
    sum1 *= input1Scale; \\\n\
    sum2 *= input1Scale; \\\n\
    sum3 *= input1Scale; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_F16_QINT_TO_F16(U8, vxc_uchar16)\n\
GEMM_F16_QINT_TO_F16(I8, vxc_char16)\n\
\n\
#define GEMM_F16_QINT16_TO_F16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toF16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    half4 valC; \\\n\
    vxc_short8 srcA, outC; \\\n\
    vxc_half8 tmpA; \\\n\
    read_type srcB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
    short in1_zp; \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA1, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA2, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA3, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_F16_QINT16_TO_F16(I16, vxc_short8)\n\
"; /* end of matrixmul_f16u8_f16_vx*/

static const char matrixmul_f16u8_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16U8MulZptoFp32_4x4;\n\
_viv_uniform float in1outScale;\n\
\n\
#define GEMM_F16_QINT_TO_QINT(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##to##src1_type_name(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    read_type srcB, outC; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempZp; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempZp, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16U8MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempZp.x; \\\n\
        sum1 += tempA1 + tempZp.y; \\\n\
        sum2 += tempA2 + tempZp.z; \\\n\
        sum3 += tempA3 + tempZp.w; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_F16_QINT_TO_QINT(U8, vxc_uchar16)\n\
GEMM_F16_QINT_TO_QINT(I8, vxc_char16)\n\
"; /* end of matrixmul_f16u8_u8_vx*/

static const char matrixmul_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniI16MulI16SumtoI32_16x1;\n\
_viv_uniform VXC_512Bits uniI16MulI16SumtoI32B_16x1;\n\
_viv_uniform float inout_beta;\n\
_viv_uniform float inout_scale;\n\
\n\
#define GEMM_QINT_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read_type srcA, srcB, outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_QINT_TO_QINT(I16, vxc_short8)\n\
\n\
__kernel void gemm_transb_I16I16toI16(image2d_array_t inputA,\n\
        image2d_array_t inputB, image2d_array_t output,\n\
        int transposeA, int transposeB, int adjointA, int adjointB,\n\
        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_short8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_short8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_int4 iVal;\n\
        vxc_float4 fpVal;\n\
        VXC_DP16x1(iVal, srcA0, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA0, srcB1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA0, srcB2, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA0, srcB3, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        sum0 = sum0 + fpVal * inout_scale + inout_beta;\n\
\n\
        VXC_DP16x1(iVal, srcA1, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA1, srcB1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA1, srcB2, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA1, srcB3, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        sum1 = sum1 + fpVal * inout_scale + inout_beta;\n\
\n\
        VXC_DP16x1(iVal, srcA2, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA2, srcB1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA2, srcB2, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA2, srcB3, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        sum2 = sum2 + fpVal * inout_scale + inout_beta;\n\
\n\
        VXC_DP16x1(iVal, srcA3, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA3, srcB1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA3, srcB2, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        VXC_DP16x1(iVal, srcA3, srcB3, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32_16x1);\n\
        VXC_DP16x1(fpVal, iVal, srcB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniI16MulI16SumtoI32B_16x1);\n\
        sum3 = sum3 + fpVal * inout_scale + inout_beta;\n\
    }\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    vxc_short8 valDst;\n\
    tmpOut0 = convert_int4_rte(sum0);\n\
    tmpOut1 = convert_int4_rte(sum1);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    tmpOut0 = convert_int4_rte(sum2);\n\
    tmpOut1 = convert_int4_rte(sum3);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of matrixmul_i16_vx*/

static const char matrixmul_merge_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float mulKIn0In1Zp;\n\
_viv_uniform float inOutScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
_viv_uniform int outer;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8U8toFp32Block4_4x4;\n\
_viv_uniform VXC_512Bits uniGemmU8U8MulZptoFp32_8x4;\n\
\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Hi_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
\n\
#define GEMM_QINT_TO_QINT_MERGE(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name##_merge( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    read_type srcA0, srcA1, srcA2, srcA3, srcB, outC; \\\n\
    vxc_float4 sum = (vxc_float4)(mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp); \\\n\
    for(int i = 0; i < outer; i++) \\\n\
    { \\\n\
        vxc_float4 sum0 = sum, sum1 = sum, sum2 = sum, sum3 = sum; \\\n\
        int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? i : get_global_id(2)), 0); \\\n\
        int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? i : get_global_id(2)), 0); \\\n\
        int8 inputA_desc, inputB_desc, output_desc; \\\n\
        _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
        int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
        _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
        _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
        int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
        _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
        for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
        { \\\n\
            vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
            vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
            VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                        VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                        VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                        VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord_a.x += 4; coord_b.y += 4; \\\n\
            VXC_DP4x4(tempA0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8toFp32Block4_4x4); \\\n\
            VXC_DP4x4(tempA1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8toFp32Block4_4x4); \\\n\
            VXC_DP4x4(tempA2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8toFp32Block4_4x4); \\\n\
            VXC_DP4x4(tempA3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8toFp32Block4_4x4); \\\n\
            VXC_DP8x4(tempB0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8MulZptoFp32_8x4); \\\n\
            VXC_DP8x4(tempB1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8MulZptoFp32_8x4); \\\n\
            VXC_DP8x4(tempB2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8MulZptoFp32_8x4); \\\n\
            VXC_DP8x4(tempB3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniGemmU8U8MulZptoFp32_8x4); \\\n\
            sum0 += tempA0 + tempB0; \\\n\
            sum1 += tempA1 + tempB1; \\\n\
            sum2 += tempA2 + tempB2; \\\n\
            sum3 += tempA3 + tempB3; \\\n\
        } \\\n\
        vxc_int4 tmpOut0, tmpOut1; \\\n\
        coord_b.y = get_global_id(1); \\\n\
        coord_b.z = get_global_id(2) + i * get_global_size(2); \\\n\
        _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
        int baseAddr = (int)coord_b.z * output_desc.s4 + output_desc.s0; \\\n\
        _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
        tmpOut0 = convert_int4_rte(sum0 * inOutScale + output_ZP); \\\n\
        tmpOut1 = convert_int4_rte(sum1 * inOutScale + output_ZP); \\\n\
        VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_b.y++; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_b.y++; \\\n\
        tmpOut0 = convert_int4_rte(sum2 * inOutScale + output_ZP); \\\n\
        tmpOut1 = convert_int4_rte(sum3 * inOutScale + output_ZP); \\\n\
        VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_b.y++; \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
GEMM_QINT_TO_QINT_MERGE(U8, vxc_uchar16)\n\
GEMM_QINT_TO_QINT_MERGE(I8, vxc_char16)\n\
\n\
#if (VX_VERSION==2)\n\
__kernel void gemm_F16F16toF16_merge(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    for(int i = 0; i < outer; i++)\n\
    {\n\
        int4 coord_a = (int4)(0, gidy, (ac2zero ? i : get_global_id(2)), 0);\n\
        int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? i : get_global_id(2)), 0);\n\
\n\
        half4 valC;\n\
        vxc_short8 srcA0, srcA1, srcA2, srcA3, outC;\n\
        vxc_half8 tmpA0, tmpA1, tmpA2, tmpA3;\n\
        vxc_short16 srcB;\n\
        vxc_half16 tmpB;\n\
        vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
        vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
        int8 inputA_desc, inputB_desc, output_desc;\n\
        _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
        int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
        _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
        _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
        int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
        _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
        for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
        {\n\
            vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
            VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                        VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                        VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord_a.x += 4; coord_b.y += 4;\n\
            _viv_asm(COPY, tmpA0, srcA0, 16);\n\
            _viv_asm(COPY, tmpA1, srcA1, 16);\n\
            _viv_asm(COPY, tmpA2, srcA2, 16);\n\
            _viv_asm(COPY, tmpA3, srcA3, 16);\n\
            _viv_asm(COPY, tmpB.hi, srcB.hi, 16);\n\
            _viv_asm(COPY, tmpB.lo, srcB.lo, 16);\n\
            VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmU8F16toF32Lo_4x4b);\n\
            VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmU8F16toF32Lo_4x4b);\n\
            VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, tmpA2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmU8F16toF32Lo_4x4b);\n\
            VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, tmpA3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmU8F16toF32Lo_4x4b);\n\
            sum0 += (tempA0);\n\
            sum1 += (tempA1);\n\
            sum2 += (tempA2);\n\
            sum3 += (tempA3);\n\
        }\n\
        coord_b.y = gidy;\n\
        coord_b.z = get_global_id(2) + i * get_global_size(2);\n\
        _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
        int baseAddr = (int)coord_b.z * output_desc.s4 + output_desc.s0;\n\
        _viv_asm(MOV, coord_b.w, baseAddr);\n\
        _viv_asm(CONV, valC, sum0);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_b.y++;\n\
        _viv_asm(CONV, valC, sum1);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_b.y++;\n\
        _viv_asm(CONV, valC, sum2);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_b.y++;\n\
        _viv_asm(CONV, valC, sum3);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
#else\n\
__kernel void gemm_F16F16toF16_merge(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    for(int i = 0; i < outer; i++)\n\
    {\n\
        int4 coord_a = (int4)(0, gidy, (ac2zero ? i : get_global_id(2)), 0);\n\
        int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? i : get_global_id(2)), 0);\n\
\n\
        half4 valC;\n\
        vxc_short8 srcA0, srcB0, srcA1, srcB1, outC;\n\
        vxc_half8 tmpA0, tmpB0, tmpA1, tmpB1;\n\
        vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
        vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
        int8 inputA_desc, inputB_desc, output_desc;\n\
        _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
        int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
        _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
        _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
        int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
        _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
        for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
        {\n\
            vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
            vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
            VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                        VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                        VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                        VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
            VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                        VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord_a.x += 4; coord_b.y += 4;\n\
            _viv_asm(COPY, tmpA0, srcA0, 16);\n\
            _viv_asm(COPY, tmpB0, srcB0, 16);\n\
            _viv_asm(COPY, tmpA1, srcA1, 16);\n\
            _viv_asm(COPY, tmpB1, srcB1, 16);\n\
\n\
            VXC_DP4x4(tempA0, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row0Lo_4x4);\n\
            VXC_DP4x4(tempB0, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row0Hi_4x4);\n\
            VXC_DP4x4(tempA1, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row1Lo_4x4);\n\
            VXC_DP4x4(tempB1, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row1Hi_4x4);\n\
            VXC_DP4x4(tempA2, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row0Lo_4x4);\n\
            VXC_DP4x4(tempB2, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row0Hi_4x4);\n\
            VXC_DP4x4(tempA3, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row1Lo_4x4);\n\
            VXC_DP4x4(tempB3, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                        uniGemmFp16toFp32Row1Hi_4x4);\n\
            sum0 += (tempA0 + tempB0);\n\
            sum1 += (tempA1 + tempB1);\n\
            sum2 += (tempA2 + tempB2);\n\
            sum3 += (tempA3 + tempB3);\n\
        }\n\
        coord_b.y = gidy;\n\
        coord_b.z = get_global_id(2) + i * get_global_size(2);\n\
        _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
        int baseAddr = (int)coord_b.z * output_desc.s4 + output_desc.s0;\n\
        _viv_asm(MOV, coord_b.w, baseAddr);\n\
        _viv_asm(CONV, valC, sum0);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_b.y++;\n\
        _viv_asm(CONV, valC, sum1);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_b.y++;\n\
        _viv_asm(CONV, valC, sum2);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_b.y++;\n\
        _viv_asm(CONV, valC, sum3);\n\
        _viv_asm(COPY, outC, valC, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
#endif\n\
"; /* end of matrixmul_merge_vx*/

static const char matrixmul_transA_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
#define GEMM_TRANSA_QINT(src0_type_name, src1_type_name, dst_type_name, read0_type, read1_type, write_type) \\\n\
__kernel void gemm_transa_##src0_type_name##src1_type_name##to##dst_type_name( \\\n\
            image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
            int transposeA, int transposeB, int adjointA, int adjointB, \\\n\
            uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read0_type srcA; \\\n\
    read1_type srcB; \\\n\
    write_type outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_float4 sum0 = (vxc_float4)(0); \\\n\
    vxc_float4 sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0); \\\n\
    vxc_float4 sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    vxc_float4 tempA0; \\\n\
    vxc_float4 tempB0; \\\n\
 \\\n\
    for(coord_a.y = 0, coord_b.y = 0; coord_a.y < K;) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.y++; \\\n\
        coord_b.y++; \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0); \\\n\
        sum1 = (sum1 + tempA0.y * tempB0); \\\n\
        sum2 = (sum2 + tempA0.z * tempB0); \\\n\
        sum3 = (sum3 + tempA0.w * tempB0); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_TRANSA_QINT(U8, U8, U8, vxc_uchar16, vxc_uchar16, vxc_uchar16)\n\
GEMM_TRANSA_QINT(I8, I8, I8, vxc_char16, vxc_char16, vxc_char16)\n\
GEMM_TRANSA_QINT(I16, I16, I16, vxc_short8, vxc_short8, vxc_short8)\n\
GEMM_TRANSA_QINT(U8, I16, I16, vxc_uchar16, vxc_short8, vxc_short8)\n\
\n\
#define GEMM_TRANSA_INPUTB_F16(src0_type_name, read0_type) \\\n\
__kernel void gemm_transa_##src0_type_name##F16to##src0_type_name( \\\n\
                        image2d_array_t inputA, \\\n\
                        image2d_array_t inputB, \\\n\
                        image2d_array_t output, \\\n\
                                    int transposeA, \\\n\
                                    int transposeB, \\\n\
                                    int adjointA, \\\n\
                                    int adjointB, \\\n\
                        uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read0_type srcA, outC; \\\n\
    vxc_short8 srcB; \\\n\
    vxc_half8 tmpB; \\\n\
 \\\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_float4 sum0 = (vxc_float4)(0); \\\n\
    vxc_float4 sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0); \\\n\
    vxc_float4 sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    short in0_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
 \\\n\
    vxc_float4 tempA0; \\\n\
    vxc_float4 tempB0; \\\n\
 \\\n\
    for(coord_a.y = 0, coord_b.y = 0; coord_a.y < K;) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.y++; \\\n\
        coord_b.y++; \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB0,tmpB,tmpB,VXC_MODIFIER(0,3,0,VXC_RM_TowardZero,0),uniConvert1stFp16ToFp32_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0); \\\n\
        sum1 = (sum1 + tempA0.y * tempB0); \\\n\
        sum2 = (sum2 + tempA0.z * tempB0); \\\n\
        sum3 = (sum3 + tempA0.w * tempB0); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_TRANSA_INPUTB_F16(U8, vxc_uchar16)\n\
GEMM_TRANSA_INPUTB_F16(I8, vxc_char16)\n\
GEMM_TRANSA_INPUTB_F16(I16, vxc_short8)\n\
\n\
__kernel void gemm_transa_F16F16toF16(\n\
                        image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
\n\
    half4 valC;\n\
    vxc_short8 srcA, srcB, outC;\n\
    vxc_half8 tmpA, tmpB;\n\
\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    vxc_float4 tempA0;\n\
    vxc_float4 tempB0;\n\
\n\
    for(coord_a.y = 0, coord_b.y = 0; coord_a.y < K;)\n\
    {\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.y++;\n\
        coord_b.y++;\n\
        _viv_asm(COPY, tmpA, srcA, 16);\n\
        VXC_DP4x4(tempA0,tmpA,tmpA,VXC_MODIFIER(0,3,0,VXC_RM_TowardZero,0),uniConvert1stFp16ToFp32_4x4);\n\
        _viv_asm(COPY, tmpB, srcB, 16);\n\
        VXC_DP4x4(tempB0,tmpB,tmpB,VXC_MODIFIER(0,3,0,VXC_RM_TowardZero,0),uniConvert1stFp16ToFp32_4x4);\n\
\n\
        sum0 = (sum0 + tempA0.x * tempB0);\n\
        sum1 = (sum1 + tempA0.y * tempB0);\n\
        sum2 = (sum2 + tempA0.z * tempB0);\n\
        sum3 = (sum3 + tempA0.w * tempB0);\n\
    }\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(CONV, valC, sum0);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum1);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum2);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum3);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of matrixmul_transA_vx*/

static const char matrixmul_transB_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************gemm transposeB fp16 fp16 to fp16*************************/\n\
_viv_uniform VXC_512Bits uniFp16MulFp16AddtoFp32_dp8x2;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
__kernel void gemm_transb_F16F16toF16(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_short8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_short8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        _viv_asm(COPY, halfB0, srcB0, 16);\n\
        _viv_asm(COPY, halfB1, srcB1, 16);\n\
        _viv_asm(COPY, halfB2, srcB2, 16);\n\
        _viv_asm(COPY, halfB3, srcB3, 16);\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        _viv_asm(COPY, halfA0, srcA0, 16);\n\
        _viv_asm(COPY, halfA1, srcA1, 16);\n\
        _viv_asm(COPY, halfA2, srcA2, 16);\n\
        _viv_asm(COPY, halfA3, srcA3, 16);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    half4 halfDst;\n\
    vxc_short8 valDst;\n\
    _viv_asm(CONV, halfDst, sum0);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    _viv_asm(CONV, halfDst, sum1);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    _viv_asm(CONV, halfDst, sum2);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    _viv_asm(CONV, halfDst, sum3);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of matrixmul_transB_f16_vx*/

static const char matrixmul_transB_f16_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************gemm transposeB fp16 uint8 to fp16*************************/\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZptoFp16_dp2x8;\n\
_viv_uniform VXC_512Bits uniFp16MulFp16AddtoFp32_dp8x2;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
__kernel void gemm_transb_F16U8toF16(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_short8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        _viv_asm(COPY, halfA0, srcA0, 16);\n\
        _viv_asm(COPY, halfA1, srcA1, 16);\n\
        _viv_asm(COPY, halfA2, srcA2, 16);\n\
        _viv_asm(COPY, halfA3, srcA3, 16);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    half4 halfDst;\n\
    vxc_short8 valDst;\n\
    sum0 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum0);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum1 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum1);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum2 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum2);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum3 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum3);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
/***********************gemm transposeB fp16 uint8 to uint8***********************************/\n\
_viv_uniform float scaleIn2divOut;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float output_ZP;\n\
\n\
__kernel void gemm_transb_F16U8toU8(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_short8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        _viv_asm(COPY, halfA0, srcA0, 16);\n\
        _viv_asm(COPY, halfA1, srcA1, 16);\n\
        _viv_asm(COPY, halfA2, srcA2, 16);\n\
        _viv_asm(COPY, halfA3, srcA3, 16);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    vxc_uchar8 valDst;\n\
    tmpOut0 = convert_int4_rte(sum0 * scaleIn2divOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum1 * scaleIn2divOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    tmpOut0 = convert_int4_rte(sum2 * scaleIn2divOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum3 * scaleIn2divOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of matrixmul_transB_f16_mix_vx*/

static const char matrixmul_transB_u8_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************gemm transposeB uint8 uint8 to fp16*************************/\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float inScaleMul;\n\
_viv_uniform VXC_512Bits uniU8SubZptoFp16_dp2x8;\n\
_viv_uniform VXC_512Bits uniFp16MulFp16AddtoFp32_dp8x2;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
__kernel void gemm_transb_U8U8toF16(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4);\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_uchar8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        VXC_DP2x8(halfA0, srcA0, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA1, srcA1, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA2, srcA2, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA3, srcA3, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    half4 halfDst;\n\
    vxc_short8 valDst;\n\
    sum0 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum0);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum1 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum1);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum2 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum2);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum3 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum3);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
/********************gemm transposeB uint8 uint8 to uint8*************************/\n\
_viv_uniform float inScaledivOut;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float output_ZP;\n\
\n\
__kernel void gemm_transb_U8U8toU8(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4);\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_uchar8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        VXC_DP2x8(halfA0, srcA0, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA1, srcA1, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA2, srcA2, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA3, srcA3, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    vxc_uchar8 valDst;\n\
    tmpOut0 = convert_int4_rte(sum0 * inScaledivOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum1 * inScaledivOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    tmpOut0 = convert_int4_rte(sum2 * inScaledivOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum3 * inScaledivOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of matrixmul_transB_u8_mix_vx*/

static const char matrixmul_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float mulKIn0In1Zp;\n\
_viv_uniform float inOutScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8U8toFp32Block4_4x4;\n\
_viv_uniform VXC_512Bits uniGemmU8U8MulZptoFp32_8x4;\n\
\n\
#define GEMM_QINT_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    read_type srcA0, srcA1, srcA2, srcA3, srcB, outC; \\\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp), sum1 = sum0; \\\n\
    vxc_float4 sum2 = sum0, sum3 = sum0; \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP8x4(tempB0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        sum0 += tempA0 + tempB0; \\\n\
        sum1 += tempA1 + tempB1; \\\n\
        sum2 += tempA2 + tempB2; \\\n\
        sum3 += tempA3 + tempB3; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = get_global_id(1); \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * inOutScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * inOutScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * inOutScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * inOutScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_QINT_TO_QINT(U8, vxc_uchar16)\n\
GEMM_QINT_TO_QINT(I8, vxc_char16)\n\
"; /* end of matrixmul_u8_vx*/

static const char matrixmul_u8f16_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float input0Scale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Hi_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmFp16MulZptoFp32_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniGemm1stU8F16toF32Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemm2ndU8F16toF32Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemm1stU8F16toF32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemm2ndU8F16toF32Hi_4x4;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_QINT_F16_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    half4 valC; \\\n\
    read_type srcA0, srcA1; \\\n\
    vxc_short8 outC; \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : gidz), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : gidz), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4(tmpZpScale, tmpB.hi, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tmpZpScale; \\\n\
        sum1 += tempA1 + tmpZpScale; \\\n\
        sum2 += tempA2 + tmpZpScale; \\\n\
        sum3 += tempA3 + tmpZpScale; \\\n\
    } \\\n\
    sum0 *= input0Scale; \\\n\
    sum1 *= input0Scale; \\\n\
    sum2 *= input0Scale; \\\n\
    sum3 *= input0Scale; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)gidz * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr);  \\\n\
 \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT_F16_TO_F16(U8, vxc_uchar16)\n\
GEMM_QINT_F16_TO_F16(I8, vxc_char16)\n\
GEMM_QINT_F16_TO_F16(I16, vxc_short8)\n\
#else\n\
#define GEMM_QINT_F16_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    half4 valC; \\\n\
    read_type srcA0, srcA1; \\\n\
    vxc_short8 outC; \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : gidz), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : gidz), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4(tempA0, srcA0, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempB0, srcA0, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempA1, srcA0, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempB1, srcA0, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, srcA1, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempB2, srcA1, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempA3, srcA1, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempB3, srcA1, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tmpZpScale, tmpB.hi, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempB0 + tmpZpScale; \\\n\
        sum1 += tempA1 + tempB1 + tmpZpScale; \\\n\
        sum2 += tempA2 + tempB2 + tmpZpScale; \\\n\
        sum3 += tempA3 + tempB3 + tmpZpScale; \\\n\
    } \\\n\
    sum0 *= input0Scale; \\\n\
    sum1 *= input0Scale; \\\n\
    sum2 *= input0Scale; \\\n\
    sum3 *= input0Scale; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)gidz * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr);  \\\n\
 \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT_F16_TO_F16(U8, vxc_uchar16)\n\
GEMM_QINT_F16_TO_F16(I8, vxc_char16)\n\
GEMM_QINT_F16_TO_F16(I16, vxc_short8)\n\
#endif\n\
"; /* end of matrixmul_u8f16_f16_vx*/

static const char matrixmul_u8f16_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Hi_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmFp16MulZptoFp32_4x4;\n\
_viv_uniform float in0outScale;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_QINT_F16_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    read_type srcA0, srcA1, outC; \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : gidz), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : gidz), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4(tmpZpScale, tmpB.hi, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tmpZpScale; \\\n\
        sum1 += tempA1 + tmpZpScale; \\\n\
        sum2 += tempA2 + tmpZpScale; \\\n\
        sum3 += tempA3 + tmpZpScale; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in0outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in0outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in0outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in0outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#else\n\
#define GEMM_QINT_F16_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    vxc_short8 srcB; \\\n\
    vxc_half8 tmpB; \\\n\
    half4 valB; \\\n\
    read_type srcA, outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
    short in0_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB0, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB1, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB2, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB3, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#endif\n\
GEMM_QINT_F16_TO_QINT(U8, vxc_uchar16)\n\
GEMM_QINT_F16_TO_QINT(I8, vxc_char16)\n\
GEMM_QINT_F16_TO_QINT(I16, vxc_short8)\n\
\n\
"; /* end of matrixmul_u8f16_u8_vx*/

static const char matrixmul_u8i16_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
#define GEMM_QINT_TO_QINT(src0_type_name, src1_type_name, dst_type_name, read0_type, read1_type, write_type) \\\n\
__kernel void gemm_##src0_type_name##src1_type_name##to##dst_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read0_type srcA; \\\n\
    read1_type srcB; \\\n\
    write_type outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_QINT_TO_QINT(U8, I16, I16, vxc_uchar8, vxc_short8, vxc_short8)\n\
\n\
__kernel void gemm_transb_U8I16toI16(image2d_array_t inputA,\n\
        image2d_array_t inputB, image2d_array_t output,\n\
        int transposeA, int transposeB, int adjointA, int adjointB,\n\
        uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    vxc_uchar8 srcA;\n\
    vxc_short8 srcB;\n\
    vxc_short8 outC;\n\
\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, get_global_id(0), (bc2zero ? 0 : get_global_id(2)), 0);\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4);\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
        vxc_float4 sum_dot0, sum_dot1, sum_dot2, sum_dot3;\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32B_4x4);\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32B_4x4);\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32B_4x4);\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 4;\n\
        coord_b.x += 4;\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvertUint8SubZpToFp32B_4x4);\n\
\n\
        sum0 = sum0 + (vxc_float4)(dot(tempA0,tempB0),dot(tempA0,tempB1),dot(tempA0,tempB2),dot(tempA0,tempB3));\n\
        sum1 = sum1 + (vxc_float4)(dot(tempA1,tempB0),dot(tempA1,tempB1),dot(tempA1,tempB2),dot(tempA1,tempB3));\n\
        sum2 = sum2 + (vxc_float4)(dot(tempA2,tempB0),dot(tempA2,tempB1),dot(tempA2,tempB2),dot(tempA2,tempB3));\n\
        sum3 = sum3 + (vxc_float4)(dot(tempA3,tempB0),dot(tempA3,tempB1),dot(tempA3,tempB2),dot(tempA3,tempB3));\n\
    }\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP);\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, outC,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, outC,\n\
                VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP);\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, outC,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, outC,\n\
                VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of matrixmul_u8i16_i16_vx*/

static const char matrixmul_u8u8_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8U8toFp32Block4_4x4;\n\
_viv_uniform VXC_512Bits uniGemmU8U8MulZptoFp32_8x4;\n\
_viv_uniform float input01Scale;\n\
_viv_uniform float mulKIn0In1Zp;\n\
\n\
#define GEMM_QINT_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read_type srcA0, srcA1, srcA2, srcA3, srcB; \\\n\
    half4 valC; \\\n\
    vxc_short8 outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp); \\\n\
    vxc_float4 sum1 = sum0, sum2 = sum0, sum3 = sum0; \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP8x4(tempB0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        sum0 += tempA0 + tempB0; \\\n\
        sum1 += tempA1 + tempB1; \\\n\
        sum2 += tempA2 + tempB2; \\\n\
        sum3 += tempA3 + tempB3; \\\n\
    } \\\n\
    sum0 *= input01Scale; \\\n\
    sum1 *= input01Scale; \\\n\
    sum2 *= input01Scale; \\\n\
    sum3 *= input01Scale; \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT_TO_F16(U8, vxc_uchar16)\n\
GEMM_QINT_TO_F16(I8, vxc_char16)\n\
\n\
#define GEMM_QINT16_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read_type srcA, srcB; \\\n\
    half4 valC; \\\n\
    vxc_short8 outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0); \\\n\
    vxc_float4 sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0); \\\n\
    vxc_float4 sum3 = (vxc_float4)(0); \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT16_TO_F16(I16, vxc_short8)\n\
"; /* end of matrixmul_u8u8_f16_vx*/

static const char maximum_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void maximum_F16F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_F16F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord.xy, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord.xy, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    coord.z ++;\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define MAXIMUM_8BITS_QUANT_IMPL(name, dtype) \\\n\
__kernel void maximum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    dtype src0, src1, dst; \\\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Hi_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Hi_2x8); \\\n\
    dst = max(src0, src1); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_8BITS_QUANT_IMPL(U8U8toU8, vxc_uchar16)\n\
MAXIMUM_8BITS_QUANT_IMPL(I8I8toI8, vxc_char16)\n\
\n\
#define MAXIMUM_8BITS_2D_QUANT_IMPL(name, dtype) \\\n\
__kernel void maximum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    dtype src0, src1, dst; \\\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Hi_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Hi_2x8); \\\n\
    dst = max(src0, src1); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_8BITS_2D_QUANT_IMPL(U8U8toU8, vxc_uchar16)\n\
MAXIMUM_8BITS_2D_QUANT_IMPL(I8I8toI8, vxc_char16)\n\
\n\
#define MAXIMUM_QUANT_IMPL(name, src_type, copy_type, dst_type) \\\n\
__kernel void maximum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    copy_type data0, data1; \\\n\
    src_type src0, src1; \\\n\
    VXC_ReadImage2DArray(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage2DArray(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    dst_type dst0, dst1, dst; \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = max(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_QUANT_IMPL(U8U8toI16,   vxc_uchar16, vxc_uchar16, vxc_short8)\n\
MAXIMUM_QUANT_IMPL(I16I16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MAXIMUM_QUANT_IMPL(I16I16toU8,  vxc_short8,  vxc_short8,  vxc_uchar16)\n\
MAXIMUM_QUANT_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar16)\n\
MAXIMUM_QUANT_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char16)\n\
MAXIMUM_QUANT_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
\n\
#define MAXIMUM_QUANT_2D_IMPL(name, src_type, copy_type, dst_type) \\\n\
__kernel void maximum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    copy_type data0, data1; \\\n\
    src_type src0, src1; \\\n\
    VXC_ReadImage(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    dst_type dst0, dst1, dst; \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = max(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_QUANT_2D_IMPL(U8U8toI16,   vxc_uchar16, vxc_uchar16, vxc_short8)\n\
MAXIMUM_QUANT_2D_IMPL(I16I16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MAXIMUM_QUANT_2D_IMPL(I16I16toU8,  vxc_short8,  vxc_short8,  vxc_uchar16)\n\
MAXIMUM_QUANT_2D_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar16)\n\
MAXIMUM_QUANT_2D_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char16)\n\
MAXIMUM_QUANT_2D_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
"; /* end of maximum_0_vx*/

static const char maximum_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
\n\
#define MAXIMUM_F16TOQUANT_IMPL(name, src0_type, copy_type, dst_type) \\\n\
__kernel void maximum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    copy_type data0; \\\n\
    src0_type src0; \\\n\
    vxc_half8 src1; \\\n\
    vxc_short8 data1; \\\n\
    dst_type dst0, dst1, dst; \\\n\
    VXC_ReadImage2DArray(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage2DArray(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = max(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_F16TOQUANT_IMPL(U8F16toU8,   vxc_uchar16, vxc_uchar16, vxc_uchar8)\n\
MAXIMUM_F16TOQUANT_IMPL(I8F16toI8,   vxc_char16,  vxc_char16,  vxc_char8)\n\
MAXIMUM_F16TOQUANT_IMPL(I16F16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MAXIMUM_F16TOQUANT_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
MAXIMUM_F16TOQUANT_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char8)\n\
MAXIMUM_F16TOQUANT_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar8)\n\
\n\
#define MAXIMUM_F16TOQUANT_2D_IMPL(name, src0_type, copy_type, dst_type) \\\n\
__kernel void maximum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    copy_type data0; \\\n\
    src0_type src0; \\\n\
    vxc_half8 src1; \\\n\
    vxc_short8 data1; \\\n\
    dst_type dst0, dst1, dst; \\\n\
    VXC_ReadImage(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = max(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_F16TOQUANT_2D_IMPL(U8F16toU8,   vxc_uchar16, vxc_uchar16, vxc_uchar8)\n\
MAXIMUM_F16TOQUANT_2D_IMPL(I8F16toI8,   vxc_char16,  vxc_char16,  vxc_char8)\n\
MAXIMUM_F16TOQUANT_2D_IMPL(I16F16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MAXIMUM_F16TOQUANT_2D_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
MAXIMUM_F16TOQUANT_2D_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char8)\n\
MAXIMUM_F16TOQUANT_2D_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar8)\n\
\n\
#define MAXIMUM_QUANT_F16TOF16_IMPL(name, src_type) \\\n\
__kernel void maximum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    src_type vec0; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 data1, dst; \\\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                uniU8MulAndPostShift0_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_QUANT_F16TOF16_IMPL(U8F16toF16,  vxc_uchar16)\n\
MAXIMUM_QUANT_F16TOF16_IMPL(I8F16toF16,  vxc_char16)\n\
MAXIMUM_QUANT_F16TOF16_IMPL(I16F16toF16, vxc_short8)\n\
\n\
#define MAXIMUM_QUANT_F16TOF16_2D_IMPL(name, src_type) \\\n\
__kernel void maximum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type vec0; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 data1, dst; \\\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                uniU8MulAndPostShift0_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_QUANT_F16TOF16_2D_IMPL(U8F16toF16,  vxc_uchar16)\n\
MAXIMUM_QUANT_F16TOF16_2D_IMPL(I8F16toF16,  vxc_char16)\n\
MAXIMUM_QUANT_F16TOF16_2D_IMPL(I16F16toF16, vxc_short8)\n\
\n\
#define MAXIMUM_QUANTTOF16_IMPL(name, src_type) \\\n\
__kernel void maximum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    src_type vec0, vec1; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 dst; \\\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, vec0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src1, vec1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_QUANTTOF16_IMPL(U8U8toF16,   vxc_uchar16)\n\
MAXIMUM_QUANTTOF16_IMPL(I8I8toF16,   vxc_char16)\n\
MAXIMUM_QUANTTOF16_IMPL(I16I16toF16, vxc_short8)\n\
\n\
#define MAXIMUM_QUANTTOF16_2D_IMPL(name, src_type) \\\n\
__kernel void maximum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type vec0, vec1; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 dst; \\\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, vec0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src1, vec1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXIMUM_QUANTTOF16_2D_IMPL(U8U8toF16,   vxc_uchar16)\n\
MAXIMUM_QUANTTOF16_2D_IMPL(I8I8toF16,   vxc_char16)\n\
MAXIMUM_QUANTTOF16_2D_IMPL(I16I16toF16, vxc_short8)"; /* end of maximum_1_vx*/

static const char maxpool_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvF16toFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
_viv_uniform float inout_scale;\n\
_viv_uniform float inout_tail;\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
\n\
#define MAXPOOL_QINT(in_name, out_name, src_type, dst_type, max_val) \\\n\
__kernel void maxpool_##in_name##to##out_name( \\\n\
     __read_only  image2d_array_t input, \\\n\
     __write_only image2d_array_t output, \\\n\
     int stride_x, int stride_y, int pad_x, int pad_y, \\\n\
     int kernel_dia_x, int kernel_dia_y, int dilation_x, int dilation_y) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y); \\\n\
    int4 coord_in = coord_out; \\\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y); \\\n\
    for(; pos_start.x < 0;) \\\n\
    { \\\n\
        pos_start.x += dilation_x; \\\n\
    } \\\n\
    for(; pos_start.y < 0;) \\\n\
    { \\\n\
        pos_start.y += dilation_y; \\\n\
    } \\\n\
    pos_end = min(pos_end, (int2)(width, height)); \\\n\
 \\\n\
    src_type src0; \\\n\
    dst_type maxVal; \\\n\
    maxVal.x = max_val; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr_a); \\\n\
 \\\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y) \\\n\
    { \\\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord_in.x += dilation_x; \\\n\
            VXC_VertMax3_Integer(maxVal, src0, src0, maxVal, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    float4 fValTmp; \\\n\
    fValTmp.x = maxVal.x * inout_scale + inout_tail; \\\n\
    int4 i4Val = convert_int4_rte(fValTmp); \\\n\
    VXC_DP2x8(maxVal, i4Val, i4Val, VXC_MODIFIER(0, 0, 0, VXC_RM_ToNearestEven, 1), \\\n\
            uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, maxVal, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXPOOL_QINT(U8,  U8,  vxc_uchar8, vxc_uchar8, 0)\n\
MAXPOOL_QINT(I8,  I8,  vxc_char8,  vxc_char8, -128)\n\
MAXPOOL_QINT(I16, I16, vxc_short8, vxc_short8, -32768)\n\
\n\
__kernel void maxpool_F16toF16(\n\
     __read_only  image2d_array_t input,\n\
     __write_only image2d_array_t output,\n\
     int stride_x, int stride_y, int pad_x, int pad_y,\n\
     int kernel_dia_x, int kernel_dia_y, int dilation_x, int dilation_y)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y);\n\
    int4 coord_in = coord_out;\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y);\n\
    for(; pos_start.x < 0;)\n\
    {\n\
        pos_start.x += dilation_x;\n\
    }\n\
    for(; pos_start.y < 0;)\n\
    {\n\
        pos_start.y += dilation_y;\n\
    }\n\
    pos_end = min(pos_end, (int2)(width, height));\n\
\n\
    vxc_short8 data0;\n\
    vxc_half8 maxVal, src0;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr_a);\n\
    coord_in.xy = pos_start;\n\
\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, maxVal, data0, 16);\n\
\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y)\n\
    {\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;)\n\
        {\n\
            VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
            coord_in.x += dilation_x;\n\
            _viv_asm(COPY, src0, data0, 16);\n\
            VXC_VertMax3_Half(maxVal, src0, src0, maxVal, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        }\n\
    }\n\
    _viv_asm(COPY, data0, maxVal, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, data0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define MAXPOOL_F16_TO_QINT(out_name, dst_type) \\\n\
__kernel void maxpool_F16to##out_name( \\\n\
     __read_only  image2d_array_t input, \\\n\
     __write_only image2d_array_t output, \\\n\
     int stride_x, int stride_y, int pad_x, int pad_y, \\\n\
     int kernel_dia_x, int kernel_dia_y, int dilation_x, int dilation_y) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y); \\\n\
    int4 coord_in = coord_out; \\\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y); \\\n\
    for(; pos_start.x < 0;) \\\n\
    { \\\n\
        pos_start.x += dilation_x; \\\n\
    } \\\n\
    for(; pos_start.y < 0;) \\\n\
    { \\\n\
        pos_start.y += dilation_y; \\\n\
    } \\\n\
    pos_end = min(pos_end, (int2)(width, height)); \\\n\
 \\\n\
    vxc_short8 data0; \\\n\
    vxc_half8 maxVal, src0; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr_a); \\\n\
    coord_in.xy = pos_start; \\\n\
 \\\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, maxVal, data0, 16); \\\n\
 \\\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y) \\\n\
    { \\\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord_in.x += dilation_x; \\\n\
            _viv_asm(COPY, src0, data0, 16); \\\n\
            VXC_VertMax3_Half(maxVal, src0, src0, maxVal, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
    float4 fValTmp; \\\n\
    VXC_DP4x4(fValTmp, maxVal, maxVal, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniConvF16toFp32_4x4); \\\n\
    fValTmp.x = fValTmp.x * inout_scale + inout_tail; \\\n\
    int4 i4Val = convert_int4_rte(fValTmp); \\\n\
    dst_type dst; \\\n\
    VXC_DP2x8(dst, i4Val, i4Val, VXC_MODIFIER(0, 0, 0, VXC_RM_ToNearestEven, 1), \\\n\
                uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
MAXPOOL_F16_TO_QINT(U8,  vxc_uchar8)\n\
MAXPOOL_F16_TO_QINT(I8,  vxc_char8)\n\
MAXPOOL_F16_TO_QINT(I16, vxc_short8)\n\
\n\
#define MAXPOOL_QINT_TO_F16(in_name, src_type, max_val) \\\n\
__kernel void maxpool_##in_name##toF16( \\\n\
     __read_only  image2d_array_t input, \\\n\
     __write_only image2d_array_t output, \\\n\
     int stride_x, int stride_y, int pad_x, int pad_y, \\\n\
     int kernel_dia_x, int kernel_dia_y, int dilation_x, int dilation_y) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y); \\\n\
    int4 coord_in = coord_out; \\\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y); \\\n\
    for(; pos_start.x < 0;) \\\n\
    { \\\n\
        pos_start.x += dilation_x; \\\n\
    } \\\n\
    for(; pos_start.y < 0;) \\\n\
    { \\\n\
        pos_start.y += dilation_y; \\\n\
    } \\\n\
    pos_end = min(pos_end, (int2)(width, height)); \\\n\
 \\\n\
    src_type src0, maxVal; \\\n\
    maxVal.x = max_val; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr_a); \\\n\
 \\\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y) \\\n\
    { \\\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord_in.x += dilation_x; \\\n\
            VXC_VertMax3_Integer(maxVal, src0, src0, maxVal, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    float4 fValTmp; \\\n\
    fValTmp.x = maxVal.x * inout_scale + inout_tail; \\\n\
    half4 h4Val; \\\n\
    _viv_asm(CONV, h4Val, fValTmp); \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(COPY, dst, h4Val, 4); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MAXPOOL_QINT_TO_F16(U8,  vxc_uchar8, 0)\n\
MAXPOOL_QINT_TO_F16(I8,  vxc_char8,  -128)\n\
MAXPOOL_QINT_TO_F16(I16, vxc_short8, -32768)\n\
\n\
__kernel void maxpool_BF16toBF16(\n\
     __read_only  image2d_array_t input,\n\
     __write_only image2d_array_t output,\n\
     int stride_x, int stride_y, int pad_x, int pad_y,\n\
     int kernel_dia_x, int kernel_dia_y, int dilation_x, int dilation_y)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y);\n\
    int4 coord_in = coord_out;\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y);\n\
    for(; pos_start.x < 0;)\n\
    {\n\
        pos_start.x += dilation_x;\n\
    }\n\
    for(; pos_start.y < 0;)\n\
    {\n\
        pos_start.y += dilation_y;\n\
    }\n\
    pos_end = min(pos_end, (int2)(width, height));\n\
\n\
    vxc_short8 data0, val0;\n\
    float4 maxVal, src0;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr_a);\n\
    coord_in.xy = pos_start;\n\
\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    VXC_DP2x8(val0, data0, zero, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, maxVal, val0, 4);\n\
\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y)\n\
    {\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;)\n\
        {\n\
            VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
            coord_in.x += dilation_x;\n\
            VXC_DP2x8(val0, data0, zero, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
            _viv_asm(COPY, src0, val0, 4);\n\
            maxVal = max(src0, maxVal);\n\
        }\n\
    }\n\
    _viv_asm(COPY, data0, maxVal, 16);\n\
    VXC_DP2x8(val0, data0, data0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, val0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of maxpool_vx*/

static const char minimum_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void minimum_F16F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_F16F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord.xy, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord.xy, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    coord.z ++;\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define MINIMUM_8BITS_QUANT_IMPL(name, dtype) \\\n\
__kernel void minimum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    dtype src0, src1, dst; \\\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Hi_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Hi_2x8); \\\n\
    dst = min(src0, src1); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_8BITS_QUANT_IMPL(U8U8toU8, vxc_uchar16)\n\
MINIMUM_8BITS_QUANT_IMPL(I8I8toI8, vxc_char16)\n\
\n\
#define MINIMUM_8BITS_2D_QUANT_IMPL(name, dtype) \\\n\
__kernel void minimum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    dtype src0, src1, dst; \\\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Hi_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Hi_2x8); \\\n\
    dst = min(src0, src1); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_8BITS_2D_QUANT_IMPL(U8U8toU8, vxc_uchar16)\n\
MINIMUM_8BITS_2D_QUANT_IMPL(I8I8toI8, vxc_char16)\n\
\n\
#define MINIMUM_QUANT_IMPL(name, src_type, copy_type, dst_type) \\\n\
__kernel void minimum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    copy_type data0, data1; \\\n\
    src_type src0, src1; \\\n\
    VXC_ReadImage2DArray(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage2DArray(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    dst_type dst0, dst1, dst; \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = min(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_QUANT_IMPL(U8U8toI16,   vxc_uchar16, vxc_uchar16, vxc_short8)\n\
MINIMUM_QUANT_IMPL(I16I16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MINIMUM_QUANT_IMPL(I16I16toU8,  vxc_short8,  vxc_short8,  vxc_uchar16)\n\
MINIMUM_QUANT_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar16)\n\
MINIMUM_QUANT_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char16)\n\
MINIMUM_QUANT_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
\n\
#define MINIMUM_QUANT_2D_IMPL(name, src_type, copy_type, dst_type) \\\n\
__kernel void minimum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    copy_type data0, data1; \\\n\
    src_type src0, src1; \\\n\
    VXC_ReadImage(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    dst_type dst0, dst1, dst; \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = min(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_QUANT_2D_IMPL(U8U8toI16,   vxc_uchar16, vxc_uchar16, vxc_short8)\n\
MINIMUM_QUANT_2D_IMPL(I16I16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MINIMUM_QUANT_2D_IMPL(I16I16toU8,  vxc_short8,  vxc_short8,  vxc_uchar16)\n\
MINIMUM_QUANT_2D_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar16)\n\
MINIMUM_QUANT_2D_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char16)\n\
MINIMUM_QUANT_2D_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
"; /* end of minimum_0_vx*/

static const char minimum_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
\n\
#define MINIMUM_F16TOQUANT_IMPL(name, src0_type, copy_type, dst_type) \\\n\
__kernel void minimum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    copy_type data0; \\\n\
    src0_type src0; \\\n\
    vxc_half8 src1; \\\n\
    vxc_short8 data1; \\\n\
    dst_type dst0, dst1, dst; \\\n\
    VXC_ReadImage2DArray(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage2DArray(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = min(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_F16TOQUANT_IMPL(U8F16toU8,   vxc_uchar16, vxc_uchar16, vxc_uchar8)\n\
MINIMUM_F16TOQUANT_IMPL(I8F16toI8,   vxc_char16,  vxc_char16,  vxc_char8)\n\
MINIMUM_F16TOQUANT_IMPL(I16F16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MINIMUM_F16TOQUANT_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
MINIMUM_F16TOQUANT_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char8)\n\
MINIMUM_F16TOQUANT_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar8)\n\
\n\
#define MINIMUM_F16TOQUANT_2D_IMPL(name, src0_type, copy_type, dst_type) \\\n\
__kernel void minimum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    copy_type data0; \\\n\
    src0_type src0; \\\n\
    vxc_half8 src1; \\\n\
    vxc_short8 data1; \\\n\
    dst_type dst0, dst1, dst; \\\n\
    VXC_ReadImage(data0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
    VXC_ReadImage(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    dst = min(dst0, dst1); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_F16TOQUANT_2D_IMPL(U8F16toU8,   vxc_uchar16, vxc_uchar16, vxc_uchar8)\n\
MINIMUM_F16TOQUANT_2D_IMPL(I8F16toI8,   vxc_char16,  vxc_char16,  vxc_char8)\n\
MINIMUM_F16TOQUANT_2D_IMPL(I16F16toI16, vxc_short8,  vxc_short8,  vxc_short8)\n\
MINIMUM_F16TOQUANT_2D_IMPL(F16F16toI16, vxc_half8,   vxc_short8,  vxc_short8)\n\
MINIMUM_F16TOQUANT_2D_IMPL(F16F16toI8,  vxc_half8,   vxc_short8,  vxc_char8)\n\
MINIMUM_F16TOQUANT_2D_IMPL(F16F16toU8,  vxc_half8,   vxc_short8,  vxc_uchar8)\n\
\n\
#define MINIMUM_QUANT_F16TOF16_IMPL(name, src_type) \\\n\
__kernel void minimum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    src_type vec0; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 data1, dst; \\\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                uniU8MulAndPostShift0_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_QUANT_F16TOF16_IMPL(U8F16toF16,  vxc_uchar16)\n\
MINIMUM_QUANT_F16TOF16_IMPL(I8F16toF16,  vxc_char16)\n\
MINIMUM_QUANT_F16TOF16_IMPL(I16F16toF16, vxc_short8)\n\
\n\
#define MINIMUM_QUANT_F16TOF16_2D_IMPL(name, src_type) \\\n\
__kernel void minimum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type vec0; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 data1, dst; \\\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(data1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, data1, 16); \\\n\
 \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                uniU8MulAndPostShift0_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_QUANT_F16TOF16_2D_IMPL(U8F16toF16,  vxc_uchar16)\n\
MINIMUM_QUANT_F16TOF16_2D_IMPL(I8F16toF16,  vxc_char16)\n\
MINIMUM_QUANT_F16TOF16_2D_IMPL(I16F16toF16, vxc_short8)\n\
\n\
#define MINIMUM_QUANTTOF16_IMPL(name, src_type) \\\n\
__kernel void minimum_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    src_type vec0, vec1; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 dst; \\\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, vec0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src1, vec1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_QUANTTOF16_IMPL(U8U8toF16,   vxc_uchar16)\n\
MINIMUM_QUANTTOF16_IMPL(I8I8toF16,   vxc_char16)\n\
MINIMUM_QUANTTOF16_IMPL(I16I16toF16, vxc_short8)\n\
\n\
#define MINIMUM_QUANTTOF16_2D_IMPL(name, src_type) \\\n\
__kernel void minimum_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output  \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type vec0, vec1; \\\n\
    vxc_half8 src0, src1; \\\n\
    vxc_short8 dst; \\\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(src0, vec0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src1, vec1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
 \\\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
 \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
MINIMUM_QUANTTOF16_2D_IMPL(U8U8toF16,   vxc_uchar16)\n\
MINIMUM_QUANTTOF16_2D_IMPL(I8I8toF16,   vxc_char16)\n\
MINIMUM_QUANTTOF16_2D_IMPL(I16I16toF16, vxc_short8)"; /* end of minimum_1_vx*/

static const char mod_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertFstToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecToFp32_4x4;\n\
\n\
_viv_uniform float in_scale0;\n\
_viv_uniform float in_scale1;\n\
_viv_uniform float out_scale;\n\
_viv_uniform float in0Tail;\n\
_viv_uniform float in1Tail;\n\
_viv_uniform float out_zp;\n\
\n\
#define MOD_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                      IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    save_type data; \\\n\
    read_type read_data0, read_data1; \\\n\
    copy_type tmpData0, tmpData1; \\\n\
    vxc_float4 in0Val1, in0Val2, in1Val1, in1Val2; \\\n\
    vxc_float4 tmpVal1, tmpVal2; \\\n\
    dst_type tmpOut1, tmpOut2; \\\n\
    read_fun(read_data0, input0, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmpData0, read_data0, 16); \\\n\
    read_fun(read_data1, input1, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmpData1, read_data1, 16); \\\n\
    VXC_DP4x4(in0Val1, tmpData0, tmpData0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
    VXC_DP4x4(in0Val2, tmpData0, tmpData0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
    VXC_DP4x4(in1Val1, tmpData1, tmpData1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
    VXC_DP4x4(in1Val2, tmpData1, tmpData1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
    in0Val1 = in0Val1 * IN0_SCALE + IN0_TAIL; \\\n\
    in0Val2 = in0Val2 * IN0_SCALE + IN0_TAIL; \\\n\
    in1Val1 = in1Val1 * IN1_SCALE + IN1_TAIL; \\\n\
    in1Val2 = in1Val2 * IN1_SCALE + IN1_TAIL; \\\n\
    if (isfmod) \\\n\
    { \\\n\
    tmpVal1 = fmod(in0Val1, in1Val1) * OUT_SCALE + OUT_OFFSET; \\\n\
    tmpVal2 = fmod(in0Val2, in1Val2) * OUT_SCALE + OUT_OFFSET; \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
    tmpVal1 = (in0Val1 - in1Val1 * floor(in0Val1 / in1Val1)) * OUT_SCALE + OUT_OFFSET; \\\n\
    tmpVal2 = (in0Val2 - in1Val2 * floor(in0Val2 / in1Val2)) * OUT_SCALE + OUT_OFFSET; \\\n\
    } \\\n\
    _viv_asm(conv_mode, tmpOut1, tmpVal1); \\\n\
    _viv_asm(conv_mode, tmpOut2, tmpVal2); \\\n\
    VXC_DP2x8(data, tmpOut1, tmpOut2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
\n\
#define TENSOR_MOD(src0_name, src1_name, dst_name, dst_type, save_type, read_type, copy_type, \\\n\
                    conv_mode, IN0_SCALE, IN0_TAIL, IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET) \\\n\
__kernel void mod_##src0_name##src1_name##to##dst_name \\\n\
    ( \\\n\
    image2d_array_t input0, \\\n\
    image2d_array_t input1, \\\n\
    image2d_array_t output, \\\n\
    int             isfmod \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    MOD_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                  IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
\n\
TENSOR_MOD(F16, F16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_half8, CONV, 1, 0, 1, 0, 1, 0)\n\
TENSOR_MOD(F16, F16, I16, short4, vxc_short8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_MOD(F16, F16, I8,  char4, vxc_char8, vxc_short8,\\\n\
                vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_MOD(F16, F16, U8,  uchar4, vxc_uchar8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
\n\
TENSOR_MOD(I16, I16, I16, short4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_MOD(I16, I16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_MOD(I8, I8, I8, char4, vxc_char8, vxc_char16,\\\n\
                vxc_char16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_MOD(I8, I8, F16, half4, vxc_short8, vxc_char16,\\\n\
                vxc_char16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_MOD(U8, U8, U8,  uchar4, vxc_uchar8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_MOD(U8, U8, F16, half4, vxc_short8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
\n\
#define TENSOR_MOD_2D(src0_name, src1_name, dst_name, dst_type, save_type, read_type, copy_type, \\\n\
    conv_mode, IN0_SCALE, IN0_TAIL, IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET) \\\n\
__kernel void mod_##src0_name##src1_name##to##dst_name##_2D \\\n\
    ( \\\n\
    image2d_array_t input0, \\\n\
    image2d_array_t input1, \\\n\
    image2d_array_t output, \\\n\
    int             isfmod \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    MOD_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                  IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, VXC_ReadImage, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
TENSOR_MOD_2D(F16, F16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_half8, CONV, 1, 0, 1, 0, 1, 0)\n\
TENSOR_MOD_2D(F16, F16, I16, short4, vxc_short8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_MOD_2D(F16, F16, I8,  char4, vxc_char8, vxc_short8,\\\n\
                vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
TENSOR_MOD_2D(F16, F16, U8,  uchar4, vxc_uchar8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
\n\
TENSOR_MOD_2D(I16, I16, I16, short4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_MOD_2D(I16, I16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_MOD_2D(I8, I8, I8, char4, vxc_char8, vxc_char16,\\\n\
                vxc_char16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_MOD_2D(I8, I8, F16, half4, vxc_short8, vxc_char16,\\\n\
                vxc_char16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
TENSOR_MOD_2D(U8, U8, U8,  uchar4, vxc_uchar8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_MOD_2D(U8, U8, F16, half4, vxc_short8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define MOD_BF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 read_data0, read_data1, vec0; \\\n\
    vxc_float4 in0Val1, in0Val2, in1Val1, in1Val2; \\\n\
    vxc_float4 tmpVal1, tmpVal2; \\\n\
    vxc_ushort8 dst0, dst1; \\\n\
    vxc_ushort8 vect; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    read_fun(read_data0, input0, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(vec0, read_data0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, in0Val1, vec0, 16); \\\n\
    VXC_DP2x8(vec0, read_data0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, in0Val2, vec0, 16); \\\n\
    read_fun(read_data1, input1, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(vec0, read_data1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, in1Val1, vec0, 16); \\\n\
    VXC_DP2x8(vec0, read_data1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, in1Val2, vec0, 16); \\\n\
    tmpVal1 = fmod(in0Val1, in1Val1); \\\n\
    tmpVal2 = fmod(in0Val2, in1Val2); \\\n\
    _viv_asm(COPY, dst0, tmpVal1, 16); \\\n\
    _viv_asm(COPY, dst1, tmpVal2, 16); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void mod_BF16BF16toBF16\n\
    (\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    int             isfmod\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    MOD_BF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray);\n\
}\n\
\n\
__kernel void mod_BF16BF16toBF16_2D\n\
    (\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    int             isfmod\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    MOD_BF16_PROCESS(VXC_ReadImage, VXC_WriteImage);\n\
}"; /* end of mod_vx*/

static const char moments_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform float zpScaleSqr_i16;\n\
_viv_uniform float zpScale2_i16;\n\
_viv_uniform float sumScale_i16;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define MOMENTS_AXIS0_QINT(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toF16( \\\n\
    image2d_array_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(0, gidy, gidz, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
 \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidy, gidz); \\\n\
    half4 tmpData; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpData, mean_vari0); \\\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
 \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
MOMENTS_AXIS0_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS0_QINT(I8, vxc_char16)\n\
\n\
#define MOMENTS_AXIS0_QINT_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toF16_2D( \\\n\
    image2d_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int2 coord = (int2)(0, gidy); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
 \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidy, 0); \\\n\
    half4 tmpData; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpData, mean_vari0); \\\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
 \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS0_QINT_2D(U8, vxc_uchar16)\n\
MOMENTS_AXIS0_QINT_2D(I8, vxc_char16)\n\
\n\
__kernel void moments_axis0_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(0, gidy, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 sumsqr0;\n\
    vxc_float4 mean_vari0 = (vxc_float4)(0);\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP8x2(sumsqr0, in_h0, in_h0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        mean_vari0 += sumsqr0;\n\
    }\n\
\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_F16toF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int2 coord = (int2)(0, gidy);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 sumsqr0;\n\
    vxc_float4 mean_vari0 = (vxc_float4)(0);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP8x2(sumsqr0, in_h0, in_h0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        mean_vari0 += sumsqr0;\n\
    }\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, 0);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(0, gidy, gidz, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr0;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0;\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP8x2(sumsqr0, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        tmpSum += sumsqr0.x;\n\
        sqr += (sumsqr0.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr0.x);\n\
    }\n\
    sum = tmpSum * input_scale + sumScale_i16;\n\
\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0);\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_I16toF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int2 coord = (int2)(0, gidy);\n\
    vxc_short8 src0;\n\
    float4 sumsqr0;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0;\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP8x2(sumsqr0, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        tmpSum += sumsqr0.x;\n\
        sqr += (sumsqr0.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr0.x);\n\
    }\n\
    sum = tmpSum * input_scale + sumScale_i16;\n\
\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0);\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, 0);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_BF16toBF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(0, gidy, gidz, 0);\n\
    vxc_ushort8 src0, src1;\n\
    vxc_ushort8 val;\n\
    vxc_float4 mean_vari0 = (vxc_float4)(0);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_float4 one = (vxc_float4)(1.0f, 1.0f, 1.0f, 1.0f);\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage2DArray(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        float4 vec0, vec1;\n\
        VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, vec0, src0, 16);\n\
        VXC_DP2x8(src1, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, vec1, src1, 16);\n\
        mean_vari0.x += dot(vec0, one) + dot(vec1, one);\n\
        mean_vari0.y += dot(vec0, vec0) + dot(vec1, vec1);\n\
    }\n\
\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
\n\
    vxc_short8 dst;\n\
    _viv_asm(COPY, src0, mean_vari0, 16);\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_BF16toBF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int2 coord = (int2)(0, gidy);\n\
    vxc_ushort8 src0, src1;\n\
    vxc_ushort8 val;\n\
    vxc_float4 mean_vari0 = (vxc_float4)(0);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_float4 one = (vxc_float4)(1.0f, 1.0f, 1.0f, 1.0f);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        float4 vec0, vec1;\n\
        VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, vec0, src0, 16);\n\
        VXC_DP2x8(src1, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, vec1, src1, 16);\n\
        mean_vari0.x += dot(vec0, one) + dot(vec1, one);\n\
        mean_vari0.y += dot(vec0, vec0) + dot(vec1, vec1);\n\
    }\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, 0);\n\
\n\
    vxc_short8 dst;\n\
    _viv_asm(COPY, src0, mean_vari0, 16);\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of moments_axis0_vx*/

static const char moments_axis01_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform float zpScaleSqr_i16;\n\
_viv_uniform float zpScale2_i16;\n\
_viv_uniform float sumScale_i16;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
#define MOMENTS_AXIS01_QINT(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toF16( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(gidz, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        half4 tmpMean, tmpVari; \\\n\
        vxc_half8 tmpVal; \\\n\
        vxc_short8 dst; \\\n\
        _viv_asm(CONV, tmpMean, mean); \\\n\
        _viv_asm(CONV, tmpVari, vari); \\\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari, \\\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
        _viv_asm(COPY, dst, tmpVal, 16); \\\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS01_QINT(I8, vxc_char16)\n\
\n\
#define MOMENTS_AXIS01_QINT_2D(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toF16_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        half4 tmpMean, tmpVari; \\\n\
        vxc_half8 tmpVal; \\\n\
        vxc_short8 dst; \\\n\
        _viv_asm(CONV, tmpMean, mean); \\\n\
        _viv_asm(CONV, tmpVari, vari); \\\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari, \\\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
        _viv_asm(COPY, dst, tmpVal, 16); \\\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT_2D(U8, vxc_uchar16)\n\
MOMENTS_AXIS01_QINT_2D(I8, vxc_char16)\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_F16toF16(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0.0f;\n\
        float sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_F16toF16_2D(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0.0f;\n\
        float sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_I16toF16(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        float tmpSum = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSum += sumsqr.x;\n\
            sqr += (sumsqr.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr.x);\n\
        }\n\
        sum += tmpSum * input_scale + sumScale_i16;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0.0f;\n\
        sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_I16toF16_2D(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        float tmpSum = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSum += sumsqr.x;\n\
            sqr += (sumsqr.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr.x);\n\
        }\n\
        sum += tmpSum * input_scale + sumScale_i16;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0.0f;\n\
        sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean_vari;\n\
        mean_vari.x = sum * dimRatio;\n\
        mean_vari.y = sqr * dimRatio;\n\
        mean_vari.y = mean_vari.y - mean_vari.x * mean_vari.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean_vari);\n\
        //_viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpMean,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of moments_axis01_vx*/

static const char moments_axis012_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform float zpScaleSqr_i16;\n\
_viv_uniform float zpScale2_i16;\n\
_viv_uniform float sumScale_i16;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define MOMENTS_AXIS012_QINT(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_##src0_type_name##toF16( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int4 coord = (int4)(gidx, 0, 0, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
        { \\\n\
            int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
            for(coord.y = 0; coord.y < height;) \\\n\
            { \\\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
                coord.y++; \\\n\
                VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
                tmpSum += (tmpSum1); \\\n\
                VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
                tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
            } \\\n\
            sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
            sum += (tmpSum + sumInZp) * input_scale; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        half4 tmpMean, tmpVari; \\\n\
        vxc_half8 tmpVal; \\\n\
        vxc_short8 dst; \\\n\
        _viv_asm(CONV, tmpMean, mean); \\\n\
        _viv_asm(CONV, tmpVari, vari); \\\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
        _viv_asm(COPY, dst, tmpVal, 16); \\\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
\n\
MOMENTS_AXIS012_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS012_QINT(I8, vxc_char16)\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
        {\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
                coord.y++;\n\
                _viv_asm(COPY, in_h, src0, 16);\n\
                VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
                tmpSumSqr += sumsqr;\n\
            }\n\
        }\n\
    }\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = (float)(0);\n\
        float sqr = (float)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
        {\n\
            float tmpSum = 0;\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
                coord.y++;\n\
                VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
                tmpSum += sumsqr.x;\n\
                sqr += (sumsqr.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr.x);\n\
            }\n\
            sum += tmpSum * input_scale + sumScale_i16;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = (float)(0);\n\
        sqr = (float)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_BF16toBF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    vxc_float4 sumsqr;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    float tmpSum = 0;\n\
    float tmpSqr = 0;\n\
    vxc_ushort8 src0, src1;\n\
    vxc_ushort8 val;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_float4 one = (vxc_float4)(1.0f, 1.0f, 1.0f, 1.0f);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
        {\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                VXC_ReadImage2DArray(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
                coord.y++;\n\
                float4 vec0, vec1;\n\
                VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
                _viv_asm(COPY, vec0, src0, 16);\n\
                VXC_DP2x8(src1, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part1_2x8);\n\
                _viv_asm(COPY, vec1, src1, 16);\n\
                tmpSum += dot(vec0, one) + dot(vec1, one);\n\
                tmpSqr += dot(vec0, vec0) + dot(vec1, vec1);\n\
            }\n\
        }\n\
    }\n\
    lcl_sum[lidx] = tmpSum;\n\
    lcl_sqr[lidx] = tmpSqr;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = (float)(0);\n\
        float sqr = (float)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean_vari;\n\
        mean_vari.x = sum * dimRatio;\n\
        mean_vari.y = sqr * dimRatio;\n\
        mean_vari.y = mean_vari.y  - mean_vari.x * mean_vari.x;\n\
\n\
        vxc_short8 dst;\n\
        _viv_asm(COPY, src0, mean_vari, 16);\n\
        VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of moments_axis012_vx*/

static const char moments_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float e2InScale;\n\
\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define MOMENTS_AXIS1_QINT(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toF16( \\\n\
    image2d_array_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    for(coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidz); \\\n\
    half4 tmpMean, tmpVari; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpMean, mean); \\\n\
    _viv_asm(CONV, tmpVari, vari); \\\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
MOMENTS_AXIS1_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS1_QINT(I8, vxc_char16)\n\
MOMENTS_AXIS1_QINT(I16, vxc_short8)\n\
\n\
#define MOMENTS_AXIS1_QINT_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toF16_2D( \\\n\
    image2d_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
        int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    for (coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
           uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, 0); \\\n\
    half4 tmpMean, tmpVari; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpMean, mean); \\\n\
    _viv_asm(CONV, tmpVari, vari); \\\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS1_QINT_2D(U8, vxc_uchar16)\n\
MOMENTS_AXIS1_QINT_2D(I8, vxc_char16)\n\
MOMENTS_AXIS1_QINT_2D(I16, vxc_short8)\n\
\n\
__kernel void moments_axis1_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 tmpSrc0;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP4x4(tmpSrc0, in_h0, in_h0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
        sum += tmpSrc0;\n\
        sqr += (tmpSrc0 * tmpSrc0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    half4 tmpMean, tmpVari;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpMean, mean);\n\
    _viv_asm(CONV, tmpVari, vari);\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis1_F16toF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 tmpSrc0;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP4x4(tmpSrc0, in_h0, in_h0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
        sum += tmpSrc0;\n\
        sqr += (tmpSrc0 * tmpSrc0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, 0);\n\
    half4 tmpMean, tmpVari;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpMean, mean);\n\
    _viv_asm(CONV, tmpVari, vari);\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis1_BF16toBF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_ushort8 src0;\n\
    vxc_ushort8 val;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage2DArray(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        float4 vec0;\n\
        VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, vec0, src0, 16);\n\
\n\
        sum += vec0;\n\
        sqr += (vec0 * vec0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    vxc_short8 tmpdst0, tmpdst1, dst;\n\
    _viv_asm(COPY, tmpdst0, mean, 16);\n\
    _viv_asm(COPY, tmpdst1, vari, 16);\n\
    VXC_DP2x8(dst, tmpdst0, tmpdst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis1_BF16toBF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
    vxc_ushort8 src0;\n\
    vxc_ushort8 val;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        float4 vec0;\n\
        VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, vec0, src0, 16);\n\
\n\
        sum += vec0;\n\
        sqr += (vec0 * vec0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, 0);\n\
    vxc_short8 tmpdst0, tmpdst1, dst;\n\
    _viv_asm(COPY, tmpdst0, mean, 16);\n\
    _viv_asm(COPY, tmpdst1, vari, 16);\n\
    VXC_DP2x8(dst, tmpdst0, tmpdst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of moments_axis1_vx*/

static const char moments_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define MOMENTS_AXIS2_QINT(src0_type_name, read0_type) \\\n\
__kernel void moments_axis2_##src0_type_name##toF16( \\\n\
    image2d_array_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
        int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, gidy, 0, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidy); \\\n\
    half4 tmpMean, tmpVari; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpMean, mean); \\\n\
    _viv_asm(CONV, tmpVari, vari); \\\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
MOMENTS_AXIS2_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS2_QINT(I8, vxc_char16)\n\
MOMENTS_AXIS2_QINT(I16, vxc_short8)\n\
\n\
__kernel void moments_axis2_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 tmpSrc0;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP4x4(tmpSrc0, in_h0, in_h0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
        sum += tmpSrc0;\n\
        sqr += (tmpSrc0 * tmpSrc0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    half4 tmpMean, tmpVari;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpMean, mean);\n\
    _viv_asm(CONV, tmpVari, vari);\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis2_BF16toBF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
    vxc_ushort8 src0;\n\
    vxc_ushort8 val;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        VXC_ReadImage2DArray(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        float4 vec0;\n\
        VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, vec0, src0, 16);\n\
\n\
        sum += vec0;\n\
        sqr += (vec0 * vec0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
\n\
    vxc_short8 tmpdst0, tmpdst1, dst;\n\
    _viv_asm(COPY, tmpdst0, mean, 16);\n\
    _viv_asm(COPY, tmpdst1, vari, 16);\n\
    VXC_DP2x8(dst, tmpdst0, tmpdst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of moments_axis2_vx*/

static const char moments_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform float4 output_ZP;\n\
_viv_uniform float4 outputScale;\n\
_viv_uniform float output_ZP0;\n\
_viv_uniform float outputScale0;\n\
_viv_uniform float output_ZP1;\n\
_viv_uniform float outputScale1;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
\n\
#define MOMENTS_AXIS0_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(0, gidy, gidz, gidz); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
    int8 inputA_desc; \\\n\
    _viv_asm(COPY, inputA_desc, input, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)gidz * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
    int2 coord_out = (int2)(gidy, gidz); \\\n\
    vxc_int4 tmpData = convert_int4_rte(mean_vari0 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
            uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS0_QINT_U8(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS0_QINT_U8_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toU8_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int2 coord = (int2)(0, gidy); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
    int2 coord_out = (int2)(gidy, 0); \\\n\
    vxc_int4 tmpData = convert_int4_rte(mean_vari0 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS0_QINT_U8_2D(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS01_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    int8 inputA_desc; \\\n\
    _viv_asm(COPY, inputA_desc, input, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)gidz * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(gidz, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 meanVari; \\\n\
        meanVari.x = sum * dimRatio; \\\n\
        meanVari.y = sqr * dimRatio - meanVari.x * meanVari.x; \\\n\
        vxc_int4 tmpData = convert_int4_rte(meanVari * outputScale + output_ZP); \\\n\
        VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT_U8(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS01_QINT_U8_2D(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toU8_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        sum = (0); sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 meanVari; \\\n\
        meanVari.x = sum * dimRatio; \\\n\
        meanVari.y = sqr * dimRatio - meanVari.x * meanVari.x; \\\n\
        vxc_int4 tmpData = convert_int4_rte(meanVari * outputScale + output_ZP); \\\n\
        VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT_U8_2D(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS1_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    int8 inputA_desc; \\\n\
    _viv_asm(COPY, inputA_desc, input, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)gidz * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
    VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    for(coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
    vxc_int4 tmpVal0 = convert_int4_rte(mean * outputScale0 + output_ZP0); \\\n\
    vxc_int4 tmpVal1 = convert_int4_rte(vari * outputScale1 + output_ZP1); \\\n\
    VXC_DP2x8(src0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
    int2 coord_out = (int2)(gidx, gidz); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS1_QINT_U8(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS1_QINT_U8_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toU8_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    for (coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
           uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio - mean * mean; \\\n\
    vxc_int4 tmpVal0 = convert_int4_rte(mean * outputScale0 + output_ZP0); \\\n\
    vxc_int4 tmpVal1 = convert_int4_rte(vari * outputScale1 + output_ZP1); \\\n\
    VXC_DP2x8(src0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
    int2 coord_out = (int2)(gidx, 0); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS1_QINT_U8_2D(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS2_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel void moments_axis2_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, gidy, 0, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio - mean * mean; \\\n\
    vxc_int4 tmpVal0 = convert_int4_rte(mean * outputScale0 + output_ZP0); \\\n\
    vxc_int4 tmpVal1 = convert_int4_rte(vari * outputScale1 + output_ZP1); \\\n\
    VXC_DP2x8(src0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
    int2 coord_out = (int2)(gidx, gidy); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS2_QINT_U8(U8, vxc_uchar16)\n\
"; /* end of moments_u8_vx*/

static const char moments_u8_axis012_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform float4 output_ZP;\n\
_viv_uniform float4 outputScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define MOMENTS_AXIS012_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int4 coord = (int4)(gidx, 0, 0, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
        { \\\n\
            int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
            for(coord.y = 0; coord.y < height;) \\\n\
            { \\\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
                coord.y++; \\\n\
                VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
                tmpSum += (tmpSum1); \\\n\
                VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
                tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
            } \\\n\
            sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
            sum += (tmpSum + sumInZp) * input_scale; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        sum = (0); sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 meanVari; \\\n\
        meanVari.x = sum * dimRatio; \\\n\
        meanVari.y = sqr * dimRatio - meanVari.x * meanVari.x; \\\n\
        vxc_int4 tmpData = convert_int4_rte(meanVari * outputScale + output_ZP); \\\n\
        VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS012_QINT_U8(U8, vxc_uchar16)\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_BF16toBF16(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    float tmpSum = 0;\n\
    float tmpSqr = 0;\n\
    vxc_ushort8 src0, src1;\n\
    vxc_ushort8 val;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_float4 one = (vxc_float4)(1.0f, 1.0f, 1.0f, 1.0f);\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage2DArray(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            float4 vec0, vec1;\n\
            VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
            _viv_asm(COPY, vec0, src0, 16);\n\
            VXC_DP2x8(src1, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, vec1, src1, 16);\n\
            tmpSum += dot(vec0, one) + dot(vec1, one);\n\
            tmpSqr += dot(vec0, vec0) + dot(vec1, vec1);\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSum;\n\
    lcl_sqr[lidx] = tmpSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0.0f;\n\
        float sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean_vari;\n\
        mean_vari.x = sum * dimRatio;\n\
        mean_vari.y = sqr * dimRatio;\n\
        mean_vari.y = mean_vari.y  - mean_vari.x * mean_vari.x;\n\
\n\
        vxc_short8 dst;\n\
        _viv_asm(COPY, src0, mean_vari, 16);\n\
        VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_BF16toBF16_2D(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    float tmpSum = 0;\n\
    float tmpSqr = 0;\n\
    vxc_ushort8 src0, src1;\n\
    vxc_ushort8 val;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_float4 one = (vxc_float4)(1.0f, 1.0f, 1.0f, 1.0f);\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage(val, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
\n\
            float4 vec0, vec1;\n\
            VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
            _viv_asm(COPY, vec0, src0, 16);\n\
            VXC_DP2x8(src1, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, vec1, src1, 16);\n\
            tmpSum += dot(vec0, one) + dot(vec1, one);\n\
            tmpSqr += dot(vec0, vec0) + dot(vec1, vec1);\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSum;\n\
    lcl_sqr[lidx] = tmpSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0.0f;\n\
        float sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean_vari;\n\
        mean_vari.x = sum * dimRatio;\n\
        mean_vari.y = sqr * dimRatio;\n\
        mean_vari.y = mean_vari.y  - mean_vari.x * mean_vari.x;\n\
\n\
        vxc_short8 dst;\n\
        _viv_asm(COPY, src0, mean_vari, 16);\n\
        VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of moments_u8_axis012_vx*/

static const char nearest_grid_sample_BF16_to_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
\n\
_viv_uniform VXC_512Bits uniBF16toFp32_part0_2x8;\n\
_viv_uniform VXC_512Bits uniBF16toFp32_part1_2x8;\n\
\n\
#define GRID_SAMPLE_BF16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    int4   x_idx       = convert_int4(in_x); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    int4    y_idx        = convert_int4(in_y); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 src; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
\n\
__kernel void nearest_grid_sample_BF16_BF16toBF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    vxc_short8 read_src;\n\
    VXC_DP2x8(read_src, read_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniBF16toFp32_part0_2x8);\n\
    _viv_asm(COPY, fxy0, read_src, 16);\n\
    VXC_DP2x8(read_src, read_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniBF16toFp32_part1_2x8);\n\
    _viv_asm(COPY, fxy1, read_src, 16);\n\
\n\
\n\
\n\
    GRID_SAMPLE_BF16_PROCESS();\n\
\n\
}\n\
"; /* end of nearest_grid_sample_BF16_to_BF16_vx*/

static const char nearest_grid_sample_F16_to_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniEvenBintoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniOddSubEvenBin_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
\n\
#define GRID_SAMPLE_F16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    int4   x_idx       = convert_int4(in_x); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    int4    y_idx        = convert_int4(in_y); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 src; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void nearest_grid_sample_F16_F32toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
__kernel void nearest_grid_sample_F16_U8toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_uchar16 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void nearest_grid_sample_F16_F16toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_F16_PROCESS();\n\
\n\
}\n\
"; /* end of nearest_grid_sample_F16_to_F16_vx*/

static const char nearest_grid_sample_F16_to_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
#define GRID_SAMPLE_F16_to_U8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    int4   x_idx       = convert_int4(in_x); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    int4    y_idx        = convert_int4(in_y); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 s0; \\\n\
    vxc_uchar16 result; \\\n\
    vxc_half8 src; \\\n\
    VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, s0, 16); \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    float4 dst4; \\\n\
    int4 dst; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP4x4(dst4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4); \\\n\
        dst4         = dst4 * uint8Scale + output_ZP; \\\n\
        dst     = convert_int4_rte(dst4); \\\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
        result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, s0, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, s0, 16); \\\n\
    } \\\n\
    VXC_DP4x4(dst4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4); \\\n\
    dst4         = dst4 * uint8Scale + output_ZP; \\\n\
    dst     = convert_int4_rte(dst4); \\\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void nearest_grid_sample_F16_F32toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
\n\
__kernel void nearest_grid_sample_F16_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_uchar16 read_coord;\n\
\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
\n\
__kernel void nearest_grid_sample_F16_F16toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_F16_to_U8_PROCESS();\n\
\n\
}\n\
\n\
"; /* end of nearest_grid_sample_F16_to_U8_vx*/

static const char nearest_grid_sample_I16_to_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part1_4x4;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_2x8;\n\
\n\
\n\
#define GRID_SAMPLE_I16_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    int4   x_idx       = convert_int4(in_x); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    int4    y_idx        = convert_int4(in_y); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_short8 src, dst; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
\n\
__kernel void nearest_grid_sample_I16_I16toI16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_short8 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1_scale;\n\
    fxy1 = fxy1 * input1_scale;\n\
\n\
    GRID_SAMPLE_I16_PROCESS();\n\
\n\
}\n\
"; /* end of nearest_grid_sample_I16_to_I16_vx*/

static const char nearest_grid_sample_I8_to_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
\n\
\n\
_viv_uniform float input1_scale;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_2x8;\n\
\n\
#define GRID_SAMPLE_I8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    int4   x_idx       = convert_int4(in_x); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    int4    y_idx        = convert_int4(in_y); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_char16 src, dst; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void nearest_grid_sample_I8_I8toI8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    vxc_char16 read_coord;\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1_scale;\n\
    fxy1 = fxy1 * input1_scale;\n\
\n\
    GRID_SAMPLE_I8_PROCESS();\n\
\n\
}\n\
"; /* end of nearest_grid_sample_I8_to_I8_vx*/

static const char nearest_grid_sample_U8_to_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 half_input0_wh;\n\
_viv_uniform float2 add_float_value;\n\
_viv_uniform int depth;\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniMultiplyAndPostShift_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define GRID_SAMPLE_U8_PROCESS() \\\n\
    fxy0 = fxy0 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    fxy1 = fxy1 * half_input0_wh.xyxy + add_float_value.xyxy; \\\n\
    float4 in_x        = (float4)(fxy0.xz, fxy1.xz); \\\n\
    int4   x_idx       = convert_int4(in_x); \\\n\
    float4  in_y        = (float4)(fxy0.yw, fxy1.yw); \\\n\
    int4    y_idx        = convert_int4(in_y); \\\n\
    int4 coord_in = (int4)(x_idx.x, y_idx.x, 0, 0); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc)); \\\n\
    int baseAddr = input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    vxc_uchar16 src, dst; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.y; \\\n\
    coord_in.y = y_idx.y; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.z; \\\n\
    coord_in.y = y_idx.z; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = x_idx.w; \\\n\
    coord_in.y = y_idx.w; \\\n\
    VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
    VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
    int loop = depth - 1; \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    while (coord_in.z < loop) \\\n\
    { \\\n\
        VXC_DP2x8(dst, src, multiplier, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_out.zw = coord_out.zw + (int2)(1, output_desc.s4); \\\n\
        coord_in.zw = coord_in.zw + (int2)(1, input_desc.s4); \\\n\
        coord_in.x = x_idx.x; \\\n\
        coord_in.y = y_idx.x; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.y; \\\n\
        coord_in.y = y_idx.y; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.z; \\\n\
        coord_in.y = y_idx.z; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_in.x = x_idx.w; \\\n\
        coord_in.y = y_idx.w; \\\n\
        VXC_OP4(img_load_3d, src, input0, coord_in.xywz, \\\n\
        VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    VXC_DP2x8(dst, src, multiplier, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void nearest_grid_sample_U8_F32toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
    coord_in1.z  = coord_in1.z + 4;\n\
\n\
    float4 fxy0 = read_imagef(input1, coord_in1.xy);\n\
    float4 fxy1 = read_imagef(input1, coord_in1.zw);\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
\n\
__kernel void nearest_grid_sample_U8_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_uchar16 read_coord;\n\
\n\
    VXC_ReadImage(read_coord, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    unsigned char input1ZP;\n\
    _viv_asm(COPY, input1ZP, input1_ZP, 4);\n\
\n\
    VXC_DP4x4(fxy0, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, input1ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
    fxy0 = fxy0 * input1Scale;\n\
    fxy1 = fxy1 * input1Scale;\n\
\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
\n\
__kernel void nearest_grid_sample_U8_F16toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int align_corners)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_in1 = coord_out.xyxy;\n\
\n\
    coord_in1.xz = coord_in1.xz * 2;\n\
\n\
    vxc_short8 read_val;\n\
    vxc_half8  read_coord;\n\
\n\
    VXC_ReadImage(read_val, input1, coord_in1.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, read_coord, read_val, 16);\n\
\n\
    float4 fxy0;\n\
    float4 fxy1;\n\
\n\
    VXC_DP4x4(fxy0, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part0_4x4);\n\
    VXC_DP4x4(fxy1, read_coord, read_coord, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
\n\
    GRID_SAMPLE_U8_PROCESS();\n\
\n\
}\n\
\n\
"; /* end of nearest_grid_sample_U8_to_U8_vx*/

static const char one_hot_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataConvert_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataConvert_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int depth;\n\
#define ONE_HOT_SH_IMPL(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data0, data1; \\\n\
    VXC_DP4x4(data0, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    VXC_DP4x4(data1, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_1_4x4); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        int4 d0 = data0 == coord.zzzz ? on_val : off_val; \\\n\
        int4 d1 = data1 == coord.zzzz ? on_val : off_val; \\\n\
 \\\n\
        dst_type dst; \\\n\
        VXC_DP2x8(dst, d0, d1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage2DArray(output, coord.xzyw, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        coord.z ++; \\\n\
    } while (coord.z < depth); \\\n\
}\n\
ONE_HOT_SH_IMPL(F16, F16,  vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL(F16, I16,  vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL(F16, I8,   vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL(F16, U8,   vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL(I16, F16,  vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL(I16, I16,  vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL(I16, BI16, vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL(I16, I8,   vxc_short8,  vxc_short8, vxc_uchar8)\n\
ONE_HOT_SH_IMPL(I16, U8,   vxc_short8,  vxc_short8, vxc_uchar8)\n\
ONE_HOT_SH_IMPL(I8,  F16,  vxc_char8,   vxc_char8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL(I8,  I8,   vxc_char8,   vxc_char8,  vxc_uchar8)\n\
\n\
#define ONE_HOT_SH_IMPL_2D(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(0), get_global_id(0)); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data, data0, data1; \\\n\
    VXC_DP4x4(data, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    int4 d4 = (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        coord.zw = coord.xx + (int2)(0, 1); \\\n\
        dst_type dst; \\\n\
        data0 = data.xxxx == d4 ? on_val : off_val; \\\n\
        data1 = data.yyyy == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord.zw = coord.zw + (int2)(2, 2); \\\n\
 \\\n\
        data0 = data.zzzz == d4 ? on_val : off_val; \\\n\
        data1 = data.wwww == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        d4 += 4; \\\n\
        coord.y += 4; \\\n\
    } while (coord.y < depth); \\\n\
}\n\
ONE_HOT_SH_IMPL_2D(F16, F16,  vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(F16, I16,  vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(F16, I8,   vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL_2D(F16, U8,   vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL_2D(I16, F16,  vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(I16, I16,  vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(I16, BI16, vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(I16, I8,   vxc_short8,  vxc_short8, vxc_uchar8)\n\
ONE_HOT_SH_IMPL_2D(I16, U8,   vxc_short8,  vxc_short8, vxc_uchar8)\n\
ONE_HOT_SH_IMPL_2D(I8,  F16,  vxc_char8,   vxc_char8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(I8,  I8,   vxc_char8,   vxc_char8,  vxc_uchar8)\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
#define ONE_HOT_ASYM_SH_IMPL(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data0, data1; \\\n\
    float4 v0, v1; \\\n\
    VXC_DP4x4(v0, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    VXC_DP4x4(v1, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_1_4x4); \\\n\
 \\\n\
    data0 = convert_int4(v0 * input_scale + input_tail); \\\n\
    data1 = convert_int4(v1 * input_scale + input_tail); \\\n\
    do \\\n\
    { \\\n\
        int4 d0 = data0 == coord.zzzz ? on_val : off_val; \\\n\
        int4 d1 = data1 == coord.zzzz ? on_val : off_val; \\\n\
 \\\n\
        dst_type dst; \\\n\
        VXC_DP2x8(dst, d0, d1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage2DArray(output, coord.xzyw, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        coord.z ++; \\\n\
    } while (coord.z < depth); \\\n\
}\n\
ONE_HOT_ASYM_SH_IMPL(U8, F16,  vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
ONE_HOT_ASYM_SH_IMPL(U8, U8,   vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
ONE_HOT_ASYM_SH_IMPL(U8, I8,   vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
ONE_HOT_ASYM_SH_IMPL(U8, I16,  vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
ONE_HOT_ASYM_SH_IMPL(U8, BI16, vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
\n\
#define ONE_HOT_ASYM_SH_IMPL_2D(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(0), get_global_id(0)); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data, data0, data1; \\\n\
    float4 v0; \\\n\
    VXC_DP4x4(v0, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    int4 d4 = (int4)(0, 1, 2, 3); \\\n\
    data = convert_int4(v0 * input_scale + input_tail); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        coord.zw = coord.xx + (int2)(0, 1); \\\n\
        dst_type dst; \\\n\
        data0 = data.xxxx == d4 ? on_val : off_val; \\\n\
        data1 = data.yyyy == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord.zw = coord.zw + (int2)(2, 2); \\\n\
 \\\n\
        data0 = data.zzzz == d4 ? on_val : off_val; \\\n\
        data1 = data.wwww == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        d4 += 4; \\\n\
        coord.y += 4; \\\n\
    } while (coord.y < depth); \\\n\
}\n\
ONE_HOT_ASYM_SH_IMPL_2D(U8, F16,  vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
ONE_HOT_ASYM_SH_IMPL_2D(U8, U8,   vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
ONE_HOT_ASYM_SH_IMPL_2D(U8, I8,   vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
ONE_HOT_ASYM_SH_IMPL_2D(U8, I16,  vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
ONE_HOT_ASYM_SH_IMPL_2D(U8, BI16, vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
__kernel void one_hot_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 int             suffix_sz,\n\
                 int             on_val,\n\
                 int             off_val\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(0), get_global_id(0));\n\
\n\
    vxc_ushort8 src0, src1;\n\
    vxc_ushort8 val;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 vec0, vec1;\n\
    VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, vec0, src0, 16);\n\
    VXC_DP2x8(src1, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, vec1, src1, 16);\n\
    int4 data0 = convert_int4(vec0);\n\
    int4 data1 = convert_int4(vec1);\n\
\n\
    do\n\
    {\n\
        int4 d0 = data0 == coord.zzzz ? on_val : off_val;\n\
        int4 d1 = data1 == coord.zzzz ? on_val : off_val;\n\
\n\
        vxc_short8 dst;\n\
        VXC_DP2x8(dst, d0, d1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
\n\
        VXC_WriteImage2DArray(output, coord.xzyw, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 int             suffix_sz,\n\
                 int             on_val,\n\
                 int             off_val\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_ushort8 src0, src1;\n\
    vxc_ushort8 val;\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 vec0;\n\
    VXC_DP2x8(src0, val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, vec0, src0, 16);\n\
    int4 data = convert_int4(vec0);\n\
    int4 data0, data1;\n\
    int4 d4 = (int4)(0, 1, 2, 3);\n\
    do\n\
    {\n\
        coord.zw = coord.xx + (int2)(0, 1);\n\
        vxc_short8 dst;\n\
        data0 = data.xxxx == d4 ? on_val : off_val;\n\
        data1 = data.yyyy == d4 ? on_val : off_val;\n\
\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.zw = coord.zw + (int2)(2, 2);\n\
\n\
        data0 = data.zzzz == d4 ? on_val : off_val;\n\
        data1 = data.wwww == d4 ? on_val : off_val;\n\
\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8);\n\
\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0));\n\
        d4 += 4;\n\
        coord.y += 4;\n\
    } while (coord.y < depth);\n\
}"; /* end of one_hot_vx*/

static const char poolwithargmax_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
//-------------------max pooling with argmax---------------\n\
_viv_uniform VXC_512Bits poolingEncode;\n\
_viv_uniform VXC_512Bits uniQuantInOutInt16Even_4x4;\n\
\n\
#define POOLWITHARGMAX_F16_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1, maxData, src0, src1; \\\n\
    vxc_half8 din0Fp16, din1Fp16; \\\n\
    vxc_half8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(src0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, src0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, src1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s02460246; \\\n\
    _viv_asm(COPY, maxData, maxDataVer, 16); \\\n\
    vxc_short8 one = (vxc_short8)(1, 1, 1, 1, 1, 1, 1, 1); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    din0EqualTmp = src0 == maxData ? one : zero; \\\n\
    din1EqualTmp = src1 == maxData ? one : zero; \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_F16to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_F16_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_F16to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_F16_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define POOLWITHARGMAX_F16_TO_I16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1, maxData, src0, src1; \\\n\
    vxc_half8 din0Fp16, din1Fp16; \\\n\
    vxc_half8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(src0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, src0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, src1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    VXC_DP4x4(din1, din0, din0, VXC_MODIFIER_BIN(0, 3, 0), uniQuantInOutInt16Even_4x4); \\\n\
    _viv_asm(COPY, maxData, maxDataVer, 16); \\\n\
    vxc_short8 one = (vxc_short8)(1, 1, 1, 1, 1, 1, 1, 1); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    din0EqualTmp = src0 == maxData ? one : zero; \\\n\
    din1EqualTmp = src1 == maxData ? one : zero; \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_F16to_I16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_F16_TO_I16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_F16to_I16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_F16_TO_I16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of poolwithargmax_F16_vx*/

static const char poolwithargmax_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits poolingEncode2;\n\
\n\
\n\
#define POOLWITHARGMAX_I16_TO_I16_U8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s02460246; \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8_SAME\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8_SAME_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniQuantInOutInt16Even_4x4;\n\
\n\
#define POOLWITHARGMAX_I16_TO_I16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    VXC_DP4x4(din1, din0, din0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniQuantInOutInt16Even_4x4); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_I16_TO_I16_I16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    vxc_short4 axisVal; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    VXC_DP4x4(din1, din0, din0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniQuantInOutInt16Even_4x4); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    axisVal = convert_short4(axisOut); \\\n\
    write_fun(axis, coordOut, axisVal, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_I16_I16\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_I16_I16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_I16_I16_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_I16_I16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirInt16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEndInt16Fp32_4x4;\n\
_viv_uniform float input_fl_scale_i16;\n\
_viv_uniform VXC_512Bits uniPackHalf8_2x8_2;\n\
\n\
#define POOLWITHARGMAX_I16_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s02460246; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    VXC_DP4x4(tmpVal0, din1, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, din1, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt16Fp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * input_fl_scale_i16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * input_fl_scale_i16; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackHalf8_2x8_2); \\\n\
    _viv_asm(COPY, din1, tmpPack, 16); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of poolwithargmax_I16_vx*/

static const char poolwithargmax_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float inputScale;\n\
_viv_uniform VXC_512Bits uniPackHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniU8EvenBinSubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
_viv_uniform VXC_512Bits poolingEncodeInt8_0;\n\
_viv_uniform VXC_512Bits poolingEncodeInt8_1;\n\
\n\
#define POOLWITHARGMAX_I8_TO_I8_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 din0, din1; \\\n\
    vxc_char16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_char16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    vxc_short8 tmp; \\\n\
    short zp = input_ZP; \\\n\
    VXC_DP2x8(tmp, maxDataVer, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8EvenBinSubZP_MulM_2x8); \\\n\
    vxc_char16 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(maxDataVer1, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    write_fun(tensorOut, coordOut, maxDataVer1,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_char16)(1); \\\n\
    din1EqualTmp &= (vxc_char16)(1); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncodeInt8_0); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(4, 7, 0), poolingEncodeInt8_1); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define POOLWITHARGMAX_I8_TO_I8_U8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 din0, din1; \\\n\
    vxc_char16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_char16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s02468ace02468ace; \\\n\
    write_fun(tensorOut, coordOut, maxDataVer1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_char16)(1); \\\n\
    din1EqualTmp &= (vxc_char16)(1); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncodeInt8_0); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(4, 7, 0), poolingEncodeInt8_1); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8_SAME\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8_SAME_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertEvenU8ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEvenU8SubZpToFp32_4x4;\n\
\n\
#define POOLWITHARGMAX_I8_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 din0, din1; \\\n\
    vxc_char16 maxDataVer, maxDataVer1; \\\n\
    vxc_char16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    vxc_short8 result; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    short zp = input_ZP; \\\n\
    VXC_DP4x4(tmpVal0, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8ToFp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8SubZpToFp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniPackHalf8_2x8); \\\n\
    _viv_asm(COPY, result, tmpPack, 16); \\\n\
    write_fun(tensorOut, coordOut, result,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_char16)(1); \\\n\
    din1EqualTmp &= (vxc_char16)(1); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncodeInt8_0); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(4, 7, 0), poolingEncodeInt8_1); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I8to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I8_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I8to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I8_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of poolwithargmax_I8_vx*/

static const char poolwithargmax_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
_viv_uniform VXC_512Bits uniU8EvenBinSubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniEncodeUint8_4x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
\n\
#define POOLWITHARGMAX_U8_TO_U8_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 din0, din1; \\\n\
    vxc_uchar16 maxDataVer, maxDataVer1; \\\n\
    vxc_uchar16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    maxDataVer  = max(din0, din1); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    maxDataVer  = max(maxDataVer1, maxDataVer); \\\n\
    vxc_short8 tmp; \\\n\
    uchar zp = input_ZP; \\\n\
    VXC_DP2x8(tmp, maxDataVer, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8EvenBinSubZP_MulM_2x8); \\\n\
    vxc_uchar16 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(maxDataVer1, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    write_fun(tensorOut, coordOut, maxDataVer1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_uchar16)(1); \\\n\
    din1EqualTmp &= (vxc_uchar16)(1); \\\n\
    VXC_DP4x8(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 7, 0), uniEncodeUint8_4x8); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_U8_TO_U8_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_U8_TO_U8_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSubZpUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniPackHalf2Short_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf2Short_2x8;\n\
_viv_uniform VXC_512Bits uniPackHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertEvenU8ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEvenU8SubZpToFp32_4x4;\n\
\n\
#define POOLWITHARGMAX_U8_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 din0, din1; \\\n\
    vxc_uchar16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_uchar16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    vxc_short8 result; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer,\\\n\
        maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    vxc_short4 tmpOut2, tmpOut3; \\\n\
    uchar zp = input_ZP; \\\n\
    VXC_DP4x4(tmpVal0, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8ToFp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8SubZpToFp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniPackHalf8_2x8); \\\n\
    _viv_asm(COPY, result, tmpPack, 16); \\\n\
    write_fun(tensorOut, coordOut, result, \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_uchar16)(1); \\\n\
    din1EqualTmp &= (vxc_uchar16)(1); \\\n\
    VXC_DP4x8(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 7, 0), uniEncodeUint8_4x8); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_U8to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_U8_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_U8_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_U8_TO_F16_I16_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 din0, din1; \\\n\
    vxc_uchar16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_uchar16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    vxc_short8 result, axisResult; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s02468ace02468ace; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    vxc_short4 tmpOut2, tmpOut3; \\\n\
    uchar zp = input_ZP; \\\n\
    VXC_DP4x4(tmpVal0, maxDataVer1, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertUint8ToFp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, maxDataVer1, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertSubZpUint8Fp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniPackHalf8_2x8); \\\n\
    _viv_asm(COPY, result, tmpPack, 16); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_uchar16)(1); \\\n\
    din1EqualTmp &= (vxc_uchar16)(1); \\\n\
    VXC_DP4x8(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 7, 0), uniEncodeUint8_4x8); \\\n\
    axisOut = clz(axisEncode); \\\n\
    _viv_asm(CONV, axisResult, axisOut); \\\n\
    write_fun(tensorOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisResult, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_U8to_F16_I16\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_U8_TO_F16_I16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_F16_I16_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_U8_TO_F16_I16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of poolwithargmax_U8_vx*/

static const char pow_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float input0_scale;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform float input0_tail;\n\
_viv_uniform float input1_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define POW_SH_IMPL(name, src0_type, copy0_type, src1_type, copy1_type, dst_type, copy2_type, conv_type) \\\n\
__kernel void pow_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    src0_type src0; \\\n\
    copy0_type data0; \\\n\
    src1_type src1; \\\n\
    copy1_type data1; \\\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, data0, src0, 16); \\\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, data1, src1, 16); \\\n\
    float4 x0, x1; \\\n\
    float4 y0, y1; \\\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4); \\\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4); \\\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4); \\\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4); \\\n\
    x0 = x0 * input0_scale + input0_tail; \\\n\
    x1 = x1 * input0_scale + input0_tail; \\\n\
    y0 = y0 * input1_scale + input1_tail; \\\n\
    y1 = y1 * input1_scale + input1_tail; \\\n\
    float4 s0 = sign(x0); \\\n\
    float4 s1 = sign(x1); \\\n\
    int4 t0 = convert_int4(y0) & 1; \\\n\
    int4 t1 = convert_int4(y1) & 1; \\\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0; \\\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1; \\\n\
    x0 = s0 * exp2(y0 * log2(fabs(x0))); \\\n\
    x1 = s1 * exp2(y1 * log2(fabs(x1))); \\\n\
    x0 = x0 * output_scale + output_zp; \\\n\
    x1 = x1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    _viv_asm(CONV_RTE, tmpVal0, x0); \\\n\
    _viv_asm(CONV_RTE, tmpVal1, x1); \\\n\
    dst_type dst0; \\\n\
 \\\n\
    copy2_type dst; \\\n\
    VXC_DP2x8(dst0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
POW_SH_IMPL(F16_F16toF16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(F16_F16toI16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL(F16_F16toI8,  vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL(F16_F16toU8,  vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, int4)\n\
POW_SH_IMPL(F16_I16toF16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(F16_I16toI16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL(I16_F16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(I16_F16toI16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL(I16_I16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(I16_I16toI16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL(F16_I8toF16,  vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(F16_I8toI8,   vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL(I8_F16toF16,  vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(I8_F16toI8,   vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL(I8_I8toF16,   vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(I8_I8toI8,    vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL(F16_U8toF16,  vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(F16_U8toU8,   vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, int4)\n\
POW_SH_IMPL(U8_F16toF16,  vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(U8_F16toU8,   vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, int4)\n\
POW_SH_IMPL(U8_U8toF16,   vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL(U8_U8toU8,    vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, int4)\n\
\n\
#define POW_SH_IMPL_2D(name, src0_type, copy0_type, src1_type, copy1_type, dst_type, copy2_type, conv_type) \\\n\
__kernel void pow_##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src0_type src0; \\\n\
    copy0_type data0; \\\n\
    src1_type src1; \\\n\
    copy1_type data1; \\\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, data0, src0, 16); \\\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, data1, src1, 16); \\\n\
    float4 x0, x1; \\\n\
    float4 y0, y1; \\\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4); \\\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4); \\\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4); \\\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4); \\\n\
    x0 = x0 * input0_scale + input0_tail; \\\n\
    x1 = x1 * input0_scale + input0_tail; \\\n\
    y0 = y0 * input1_scale + input1_tail; \\\n\
    y1 = y1 * input1_scale + input1_tail; \\\n\
    float4 s0 = sign(x0); \\\n\
    float4 s1 = sign(x1); \\\n\
    int4 t0 = convert_int4(y0) & 1; \\\n\
    int4 t1 = convert_int4(y1) & 1; \\\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0; \\\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1; \\\n\
    x0 = s0 * exp2(y0 * log2(fabs(x0))); \\\n\
    x1 = s1 * exp2(y1 * log2(fabs(x1))); \\\n\
    x0 = x0 * output_scale + output_zp; \\\n\
    x1 = x1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type tmpVal0, tmpVal1; \\\n\
    _viv_asm(CONV_RTE, tmpVal0, x0); \\\n\
    _viv_asm(CONV_RTE, tmpVal1, x1); \\\n\
    dst_type dst0; \\\n\
 \\\n\
    copy2_type dst; \\\n\
    VXC_DP2x8(dst0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
POW_SH_IMPL_2D(F16_F16toF16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(F16_F16toI16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL_2D(F16_F16toI8,  vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL_2D(F16_F16toU8,  vxc_short8,  vxc_half8,   vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, int4)\n\
POW_SH_IMPL_2D(F16_I16toF16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(F16_I16toI16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL_2D(I16_F16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(I16_F16toI16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL_2D(I16_I16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(I16_I16toI16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  int4)\n\
POW_SH_IMPL_2D(F16_I8toF16,  vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(F16_I8toI8,   vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL_2D(I8_F16toF16,  vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(I8_F16toI8,   vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL_2D(I8_I8toF16,   vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(I8_I8toI8,    vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  vxc_char16,  int4)\n\
POW_SH_IMPL_2D(F16_U8toF16,  vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(F16_U8toU8,   vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, int4)\n\
POW_SH_IMPL_2D(U8_F16toF16,  vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(U8_F16toU8,   vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, int4)\n\
POW_SH_IMPL_2D(U8_U8toF16,   vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_half8,   vxc_short8,  half4)\n\
POW_SH_IMPL_2D(U8_U8toU8,    vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, vxc_uchar16, int4)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void pow_BF16_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_array_t input1,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src0, src1, dst, tmpData;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, x0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, x1, tmpData, 16);\n\
\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, y0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, y1, tmpData, 16);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    _viv_asm(COPY, src0, tmpDst0, 16);\n\
    _viv_asm(COPY, src1, tmpDst1, 16);\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_BF16_BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_array_t input1,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_ushort8 src0, src1, dst, tmpData;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, x0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, x1, tmpData, 16);\n\
\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, y0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, y1, tmpData, 16);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    _viv_asm(COPY, src0, tmpDst0, 16);\n\
    _viv_asm(COPY, src1, tmpDst1, 16);\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pow_vx*/

static const char pre_process_bgra_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1Bgra_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2Bgra_4x4;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtractInt32BgraToU8_2x8;\n\
_viv_uniform VXC_512Bits uniExchangeBgra_2x8;\n\
_viv_uniform VXC_512Bits uniExchangeBgra2_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp5BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp6BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp7BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp8BgraShort_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniExtractBfromBgra_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGfromBgra_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRfromBgra_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_bgra_scale_U8toU8(\n\
         __read_only image2d_array_t  input,\n\
        __write_only image2d_array_t  output,\n\
              global int *            xRatio,\n\
              global int *            yRatio,\n\
              global int *            xOffset,\n\
              global int *            yOffset,\n\
                     float            rMean,\n\
                     float            gMean,\n\
                     float            bMean,\n\
                     float            r_scale,\n\
                     int              reverse_channel,\n\
                     int              trans,\n\
                     float            g_scale,\n\
                     float            b_scale\n\
    )\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx = (sx + (*xOffset)) * 4 ;\n\
    sy += (*yOffset);\n\
    int4 srcPos = (int4)(sx.x, sy, sy + 1, sx.y);\n\
    vxc_uchar16 lineBGRA0, lineBGRA1, lineBGRA2, lineBGRA3;\n\
    vxc_uchar16 dataB, dataG, dataR;\n\
\n\
    VXC_ReadImage(lineBGRA0, input, srcPos.xy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA0, input, srcPos.xz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(lineBGRA1, input, srcPos.wy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA1, input, srcPos.wz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    srcPos.w = sx.w;\n\
\n\
    VXC_ReadImage(lineBGRA2, input, srcPos.xy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA2, input, srcPos.xz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(lineBGRA3, input, srcPos.wy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA3, input, srcPos.wz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar4 val_u8;\n\
    int4 tmp1, tmp2, result1, result2;\n\
    float4 tmpDst, tmp0;\n\
    float4 mean = (float4)(bMean, gMean, rMean, 0);\n\
    float4 var = (float4)(b_scale, g_scale, r_scale, 0);\n\
    //tmpFx = (int4)(fx.x, fx.x, fx.x, fx.x);\n\
    int tmpV = 1 << 19;\n\
    vxc_short8 tmpFx;\n\
    VXC_DP2x8(tmpFx, fx, fx, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    //tmpFx = fx.xxxx;\n\
    VXC_DP4x4(tmp1, lineBGRA0, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA0, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    //tmpFx = fx.yyyy;\n\
    VXC_DP4x4(tmp1, lineBGRA1, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA1, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result2 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    vxc_uchar16 dst, data;\n\
    VXC_DP2x8(dst, result1, result2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInt32BgraToU8_2x8);\n\
\n\
    //tmpFx = fx.zzzz;\n\
    VXC_DP4x4(tmp1, lineBGRA2, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp5BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA2, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp6BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    //tmpFx = fx.wwww;\n\
    VXC_DP4x4(tmp1, lineBGRA3, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp7BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA3, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp8BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result2 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    VXC_DP2x8(dst, result1, result2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtractInt32BgraToU8_2x8);\n\
\n\
    VXC_DP2x8(data, dst, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExchangeBgra_2x8);\n\
    VXC_DP2x8(data, dst, dst, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0), uniExchangeBgra2_2x8);\n\
\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    dstPos.z = bOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    dstPos.z = 1;\n\
    VXC_WriteImage2DArray(output, dstPos, data.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    dstPos.z = rOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, data.s89ab, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_bgra_copy_U8toU8(\n\
         __read_only image2d_array_t  input,\n\
        __write_only image2d_array_t  output,\n\
              global int *            xRatio,\n\
              global int *            yRatio,\n\
              global int *            xOffset,\n\
              global int *            yOffset,\n\
                     float            rMean,\n\
                     float            gMean,\n\
                     float            bMean,\n\
                     float            r_scale,\n\
                     int              reverse_channel,\n\
                     int              trans,\n\
                     float            g_scale,\n\
                     float            b_scale\n\
)\n\
{\n\
    int2 pos = (int2)((get_global_id(0) + (*xOffset)) << 2, get_global_id(1) + (*yOffset));\n\
\n\
    vxc_uchar16 lineBGRA0;\n\
    float4 tmpB, tmpG, tmpR;\n\
    float4 tmpDst;\n\
    int4 result1, result2;\n\
    vxc_uchar16 dst;\n\
\n\
    VXC_ReadImage(lineBGRA0, input, pos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(tmpB, lineBGRA0, lineBGRA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBfromBgra_4x4);\n\
    VXC_DP4x4(tmpG, lineBGRA0, lineBGRA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGfromBgra_4x4);\n\
    VXC_DP4x4(tmpR, lineBGRA0, lineBGRA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRfromBgra_4x4);\n\
\n\
    tmpDst = (tmpB - bMean) * b_scale;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    tmpDst = (tmpG - gMean) * g_scale;\n\
    result2 = convert_int4_rte(tmpDst * outputScale + zp);\n\
    VXC_DP2x8(dst, result1, result2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
\n\
    int4 dstPos = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    dstPos.z = bOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    dstPos.z = 1;\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    tmpDst = (tmpR - rMean) * r_scale;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
    VXC_DP2x8(dst, result1, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
\n\
    dstPos.z = rOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_bgra_vx*/

static const char pre_process_gray_vx[] = "/*\n\
 ============================================================================\n\
 Name        : GrayScale.vx\n\
 Author      : Sam\n\
 Version     :\n\
 Copyright   : Your copyright notice\n\
 Description :\n\
 ============================================================================\n\
 */\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtactInteger_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniDataMulAlpha_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubMean_4x4;\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
\n\
__kernel void pre_process_gray_scale_U8toF16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio);\n\
\n\
    int4 xPos       = get_global_id(0);\n\
    int yPos        = get_global_id(1);\n\
\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14);\n\
    xPos += (int4)(0, 1, 2, 3);\n\
\n\
    //x\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x;\n\
    int4 sx = fx0 & 0xffff8000;\n\
    fx0 -= sx;\n\
    sx = sx >> 15;\n\
\n\
    vxc_short4 fx;\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift);\n\
    //y\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y;\n\
    int sy = fy & 0xffff8000; // Floor\n\
\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
\n\
    //R\n\
    vxc_uchar16 line0Y;\n\
    vxc_uchar16 line1Y;\n\
    int4 coord;\n\
    sx = sx + *xOffset;\n\
    coord.xyz    = sx.xyz;\n\
    coord.w        = sy + *yOffset;\n\
    int2 coord1 = (int2)(sx.w, coord.w);\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float grayMean = mean;\n\
\n\
    int4 test01, temp1;\n\
    int4 test02, temp2;\n\
    int4 tt;\n\
    vxc_uchar4 val;\n\
    int2 coord_out = (int2)(xPos.x, yPos);\n\
\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniGetTempVal);\n\
    temp1 = temp1 + test01;\n\
\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniGetTempVal);\n\
    temp2 = temp2 + test02;\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10);\n\
\n\
    VXC_DP4x4(val, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\n\
        uniExtractBytes);\n\
\n\
    //convert U8 to FP16\n\
    half f16mean;\n\
    half f16alpha;\n\
    vxc_half4    dst;\n\
    vxc_short4 tmp_dst;\n\
    _viv_asm(CONV, f16mean, grayMean);\n\
    _viv_asm(CONV, f16alpha, f32Var);\n\
    VXC_DP4x4(dst, val, f16mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniDataSubMean_4x4);\n\
    VXC_DP4x4(dst, dst, f16alpha, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniDataMulAlpha_4x4);\n\
    _viv_asm(COPY, tmp_dst, dst, 8);\n\
    VXC_WriteImage(output, coord_out, tmp_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_gray_scale_U8toI16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio);\n\
\n\
    int4 xPos        = get_global_id(0);\n\
    int yPos        = get_global_id(1);\n\
\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14);\n\
    xPos += (int4)(0, 1, 2, 3);\n\
\n\
    //x\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x;\n\
    int4 sx = fx0 & 0xffff8000;\n\
    fx0 -= sx;\n\
    sx = sx >> 15;\n\
\n\
    vxc_short4 fx;\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniAddRShift);\n\
    //y\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y;\n\
    int sy = fy & 0xffff8000; // Floor\n\
\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
\n\
    vxc_uchar16 line0Y;\n\
    vxc_uchar16 line1Y;\n\
    int4 coord;\n\
    sx = sx + *xOffset;\n\
    coord.xyz    = sx.xyz;\n\
    coord.w        = sy + *yOffset;\n\
    int2 coord1 = (int2)(sx.w, coord.w);\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float grayMean = mean * f32Var;\n\
\n\
    int4 test01, temp1;\n\
    int4 test02, temp2;\n\
    int4 tt;\n\
    vxc_uchar4 val;\n\
    int2 coord_out = (int2)(xPos.x, yPos);\n\
\n\
    vxc_uchar8 line1, line2;\n\
\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal);\n\
    temp1 = temp1 + test01;\n\
\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal);\n\
    temp2 = temp2 + test02;\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10);\n\
\n\
    vxc_float4    tmp_dst;\n\
    vxc_uchar4 u8_dst;\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\n\
        uniExtractBytes);\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\n\
        uniConvertIntergetoF32_4x4);\n\
\n\
    //convert U8 to dfp8\n\
    int4 dst0;\n\
    vxc_short4 dst;\n\
    tmp_dst = tmp_dst * f32Var - grayMean;\n\
    tmp_dst = tmp_dst * outputScale  + outputZP;\n\
    dst0 = convert_int4_rte(tmp_dst);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\n\
        uniExtactInteger_2x8);\n\
\n\
    VXC_WriteImage(output, coord_out, dst,\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define PRE_PROCESS_GRAY_SCALE_8BITS(dst_type_name, write_type) \\\n\
__kernel void pre_process_gray_scale_U8to##dst_type_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
        global int               *xRatio, \\\n\
        global int               *yRatio, \\\n\
        global int               *xOffset, \\\n\
        global int               *yOffset, \\\n\
               float             mean, \\\n\
               float             f32Var \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos        = get_global_id(0); \\\n\
    int yPos        = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz    = sx.xyz; \\\n\
    coord.w        = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float grayMean = mean * f32Var; \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int2 coord_out = (int2)(xPos.x, yPos); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    int4 dst0; \\\n\
    write_type dst; \\\n\
    tmp_dst = tmp_dst * f32Var - grayMean; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtactInteger_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
PRE_PROCESS_GRAY_SCALE_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_SCALE_8BITS(I8, vxc_char16)"; /* end of pre_process_gray_vx*/

static const char pre_process_gray_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l00_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l10_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l01_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l11_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l21_4x4;\n\
\n\
__kernel void pre_process_gray_4over3_U8toU8\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage(src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input, coord_in, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src3, input, coord_in, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.xy = (coord_in.xy >> 2) * 3;\n\
    coord_in.zw = coord_in.yy + (int2)(1, 2);\n\
\n\
    vxc_uchar16 dst0, dst1, dst2;\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    VXC_WriteImage(output, coord_in.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_in.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_in.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_gray_half_U8toU8\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar16 src0;\n\
\n\
    VXC_ReadImage(src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.xy = coord_in.xy >> 1;\n\
\n\
    VXC_WriteImage(output, coord_in.xy, src0.s02468ace, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_gray_2_vx*/

static const char pre_process_gray_copy_vx[] = "/*\n\
 ============================================================================\n\
 Name        : GrayScale.vx\n\
 Author      : Sam\n\
 Version     :\n\
 Copyright   : Your copyright notice\n\
 Description :\n\
 ============================================================================\n\
 */\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataMeanStddevLo_2x8;\n\
_viv_uniform VXC_512Bits uniDataMeanStddevHi_2x8;\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
\n\
__kernel void pre_process_gray_copy_U8toF16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    coord.xy += (int2) (*xOffset, *yOffset);\n\
    vxc_uchar16 src0;\n\
    vxc_half8   dst0, dst1;\n\
\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.x = coord.z + 8;\n\
    float4      paramData = (float4)(mean * f32Var, mean * f32Var, mean * f32Var, f32Var);\n\
    //convert U8 to FP16\n\
    half4 paramData_f16;\n\
    vxc_short8 tmp_dst;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
        uniDataMeanStddevLo_2x8);\n\
    VXC_DP2x8(dst1, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
        uniDataMeanStddevHi_2x8);\n\
    _viv_asm(COPY, tmp_dst, dst0, 16);\n\
    VXC_WriteImage(output, coord.zw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, tmp_dst, dst1, 16);\n\
    VXC_WriteImage(output, coord.xw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_gray_copy_U8toI16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    coord.xy += (int2) (*xOffset, *yOffset);\n\
    vxc_uchar16 src0;\n\
    vxc_short8  dst0, dst1;\n\
\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.x = coord.z + 8;\n\
\n\
    f32Var *= outputScale;\n\
    float4 paramData = (float4)(mean * f32Var - outputZP, mean * f32Var - outputZP,\n\
                         mean * f32Var - outputZP, f32Var);\n\
    //convert U8 to FP16\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniDataMeanStddevLo_2x8);\n\
    VXC_DP2x8(dst1, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniDataMeanStddevHi_2x8);\n\
    VXC_WriteImage(output, coord.zw, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.xw, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define PRE_PROCESS_GRAY_COPY_8BITS(dst_type_name, write_type) \\\n\
__kernel void pre_process_gray_copy_U8to##dst_type_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
        global int               *xRatio, \\\n\
        global int               *yRatio, \\\n\
        global int               *xOffset, \\\n\
        global int               *yOffset, \\\n\
               float             mean, \\\n\
               float             f32Var \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2) (*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0; \\\n\
    write_type dst; \\\n\
 \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    f32Var *= outputScale; \\\n\
    float4  paramData = (float4)(mean * f32Var - outputZP, mean * f32Var - outputZP, \\\n\
        mean * f32Var - outputZP, f32Var); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord.zw, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
PRE_PROCESS_GRAY_COPY_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_gray_copy_vx*/

static const char pre_process_nv12_copy_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar_b;\n\
_viv_uniform float outputScaleVar_g;\n\
_viv_uniform float outputScaleVar_r;\n\
\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform VXC_512Bits uniConvertNV12toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniExtractUVtoCharSub128_2x8;\n\
_viv_uniform VXC_512Bits uniExtractYtoShortSub16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUchartoFp32_4x4;\n\
\n\
#define NV12_COPY_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_nv12_copy_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t uv_img, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             nv_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int sy = gidy + (*yOffset); \\\n\
    int sx = gidx + (*xOffset); \\\n\
    int uvX = sx & 0xfffffffe; \\\n\
    int uvY = sy >> 1; \\\n\
 \\\n\
    vxc_uchar16 Y, UV; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, (int2)(sx,sy), 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(UV, uv_img,(int2)(uvX,uvY), 0,VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
  \\\n\
    if (nv_type == 1) \\\n\
    { \\\n\
        UV.s0123 = UV.s1032; \\\n\
    } \\\n\
 \\\n\
    vxc_short8 tmpY; \\\n\
    vxc_char16 tmpUV; \\\n\
    short tmpVal = 16; \\\n\
    VXC_DP2x8(tmpY, Y, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractYtoShortSub16_2x8); \\\n\
    tmpVal = 128; \\\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractUVtoCharSub128_2x8); \\\n\
 \\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    VXC_DP4x4(DstB_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
NV12_COPY_SH_IMPL(U8toU8,  vxc_uchar8, int4,  vxc_uchar8, 8)\n\
NV12_COPY_SH_IMPL(U8toI8,  vxc_char8,  int4,  vxc_char8,  8)\n\
NV12_COPY_SH_IMPL(U8toI16, vxc_short8, int4,  vxc_short8, 16)\n\
NV12_COPY_SH_IMPL(U8toF16, vxc_half8,  half4, vxc_short8, 16)\n\
"; /* end of pre_process_nv12_copy_vx*/

static const char pre_process_nv12_rggb_copy_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar_b;\n\
_viv_uniform float outputScaleVar_g;\n\
_viv_uniform float outputScaleVar_r;\n\
\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform VXC_512Bits uniConvertNV12toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniExtractUVtoCharSub128_2x8;\n\
_viv_uniform VXC_512Bits uniExtractYtoShortSub16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUchartoFp32_4x4;\n\
\n\
#define NV12_RGGB_COPY_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_nv12_rggb_copy_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t uv_img, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             nv_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int sy = gidy + (*yOffset); \\\n\
    int sx = gidx + (*xOffset); \\\n\
    int uvX = sx & 0xfffffffe; \\\n\
    int uvY = sy >> 1; \\\n\
 \\\n\
    vxc_uchar16 Y, UV; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, (int2)(sx,sy), 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(UV, uv_img,(int2)(uvX,uvY), 0,VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
  \\\n\
    if (nv_type == 3) \\\n\
    { \\\n\
        UV.s0123 = UV.s1032; \\\n\
    } \\\n\
 \\\n\
    vxc_short8 tmpY; \\\n\
    vxc_char16 tmpUV; \\\n\
    short tmpVal = 16; \\\n\
    VXC_DP2x8(tmpY, Y, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractYtoShortSub16_2x8); \\\n\
    tmpVal = 128; \\\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractUVtoCharSub128_2x8); \\\n\
 \\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    VXC_DP4x4(DstB_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dstPos.z = 2; \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
NV12_RGGB_COPY_SH_IMPL(U8toU8,  vxc_uchar8, int4,  vxc_uchar8, 8)\n\
NV12_RGGB_COPY_SH_IMPL(U8toI8,  vxc_char8,  int4,  vxc_char8,  8)\n\
NV12_RGGB_COPY_SH_IMPL(U8toI16, vxc_short8, int4,  vxc_short8, 16)\n\
NV12_RGGB_COPY_SH_IMPL(U8toF16, vxc_half8,  half4, vxc_short8, 16)\n\
"; /* end of pre_process_nv12_rggb_copy_vx*/

static const char pre_process_nv12_rggb_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar_b;\n\
_viv_uniform float outputScaleVar_g;\n\
_viv_uniform float outputScaleVar_r;\n\
\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform uint  xrIntFloat_16;\n\
_viv_uniform uint  yrIntFloat_16;\n\
\n\
_viv_uniform VXC_512Bits uniConvertNV12toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUVtoCharSub128_2x8;\n\
_viv_uniform VXC_512Bits uniConvertYtoShortSub16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateYShift_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateUVShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUchartoFp32_4x4;\n\
\n\
#define NV12_RGGB_OPT_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_nv12_rggb_scale_##name##_gq \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t uv_img, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             nv_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    uint4 gidx = get_global_id(0); \\\n\
    uint gidy = get_global_id(1); \\\n\
    gidx += (uint4)(0, 1, 2, 3); \\\n\
 \\\n\
    uint dy = (gidy * yrIntFloat_16) >> 16; \\\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16; \\\n\
    int sy = convert_int(dy) + (*yOffset); \\\n\
    int4 sx = convert_int4(dx) + (*xOffset); \\\n\
    int4 uvX = sx & 0xfffffffe; \\\n\
    int uvY = sy >> 1; \\\n\
 \\\n\
    vxc_uchar16 Y, UV; \\\n\
    int2 coord = (int2)(sx.x, sy); \\\n\
    int2 coord_uv = (int2)(uvX.x, uvY); \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    if (nv_type == 3) \\\n\
    { \\\n\
        UV.s0123456789abcdef = UV.s1032547698badcfe; \\\n\
    } \\\n\
 \\\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8}; \\\n\
    vxc_uchar16 maskShiftUv = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8}; \\\n\
    int4 offsetUV = uvX - uvX.x; \\\n\
 \\\n\
    vxc_ushort8 diffY, diffUV; \\\n\
    _viv_asm(COPY, diffY, sx, 16); \\\n\
    _viv_asm(COPY, diffUV, offsetUV, 16); \\\n\
 \\\n\
    vxc_ushort8 constData = 8; \\\n\
    VXC_DP2x8(maskShift, diffY, constData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
                uniCalculateYShift_2x8); \\\n\
    VXC_DP2x8(maskShiftUv, diffUV, constData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                uniCalculateUVShift_2x8); \\\n\
    VXC_BitExtract(Y, Y, Y, maskShift, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_BitExtract(UV, UV, UV, maskShiftUv, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_short8 tmpY; \\\n\
    vxc_char16 tmpUV; \\\n\
    short tmpVal = 16; \\\n\
    VXC_DP2x8(tmpY, Y, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertYtoShortSub16_2x8); \\\n\
    tmpVal = 128; \\\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8); \\\n\
 \\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    VXC_DP4x4(DstB_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dstPos.z = 2; \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
NV12_RGGB_OPT_SH_IMPL(U8toU8,  vxc_uchar8, int4,  vxc_uchar8, 8)\n\
NV12_RGGB_OPT_SH_IMPL(U8toI8,  vxc_char8,  int4,  vxc_char8,  8)\n\
NV12_RGGB_OPT_SH_IMPL(U8toI16, vxc_short8, int4,  vxc_short8, 16)\n\
NV12_RGGB_OPT_SH_IMPL(U8toF16, vxc_half8,  half4, vxc_short8, 16)\n\
\n\
#define NV12_RGGB_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_nv12_rggb_scale_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t uv_img, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             nv_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    uint4 gidx = get_global_id(0); \\\n\
    uint gidy = get_global_id(1); \\\n\
    gidx += (uint4)(0, 1, 2, 3); \\\n\
 \\\n\
    uint dy = (gidy * yrIntFloat_16) >> 16; \\\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16; \\\n\
    int sy = convert_int(dy) + (*yOffset); \\\n\
    int4 sx = convert_int4(dx) + (*xOffset); \\\n\
    int4 uvX = sx & 0xfffffffe; \\\n\
    int uvY = sy >> 1; \\\n\
 \\\n\
    vxc_uchar16 Y, UV; \\\n\
    int2 coord = (int2)(sx.x, sy); \\\n\
    int2 coord_uv = (int2)(uvX.x, uvY); \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x = sx.y; \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x = sx.z; \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x = sx.w; \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_uv.x = uvX.y; \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_uv.x = uvX.z; \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_uv.x = uvX.w; \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    if (nv_type == 3) \\\n\
    { \\\n\
        UV.s01234567 = UV.s10325476; \\\n\
    } \\\n\
 \\\n\
    vxc_short8 tmpY; \\\n\
    vxc_char16 tmpUV; \\\n\
    short tmpVal = 16; \\\n\
    VXC_DP2x8(tmpY, Y, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertYtoShortSub16_2x8); \\\n\
    tmpVal = 128; \\\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8); \\\n\
 \\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    VXC_DP4x4(DstB_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dstPos.z = 2; \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
NV12_RGGB_SH_IMPL(U8toU8,  vxc_uchar8, int4,  vxc_uchar8, 8)\n\
NV12_RGGB_SH_IMPL(U8toI8,  vxc_char8,  int4,  vxc_char8,  8)\n\
NV12_RGGB_SH_IMPL(U8toI16, vxc_short8, int4,  vxc_short8, 16)\n\
NV12_RGGB_SH_IMPL(U8toF16, vxc_half8,  half4, vxc_short8, 16)\n\
"; /* end of pre_process_nv12_rggb_scale_vx*/

static const char pre_process_nv12_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar_b;\n\
_viv_uniform float outputScaleVar_g;\n\
_viv_uniform float outputScaleVar_r;\n\
\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform uint  xrIntFloat_16;\n\
_viv_uniform uint  yrIntFloat_16;\n\
\n\
_viv_uniform VXC_512Bits uniConvertNV12toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUVtoCharSub128_2x8;\n\
_viv_uniform VXC_512Bits uniConvertYtoShortSub16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateYShift_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateUVShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUchartoFp32_4x4;\n\
\n\
#define NV12_OPT_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_nv12_scale_##name##_gq \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t uv_img, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             nv_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    uint4 gidx = get_global_id(0); \\\n\
    uint gidy = get_global_id(1); \\\n\
    gidx += (uint4)(0, 1, 2, 3); \\\n\
 \\\n\
    uint dy = (gidy * yrIntFloat_16) >> 16; \\\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16; \\\n\
    int sy = convert_int(dy) + (*yOffset); \\\n\
    int4 sx = convert_int4(dx) + (*xOffset); \\\n\
    int4 uvX = sx & 0xfffffffe; \\\n\
    int uvY = sy >> 1; \\\n\
 \\\n\
    vxc_uchar16 Y, UV; \\\n\
    int2 coord = (int2)(sx.x, sy); \\\n\
    int2 coord_uv = (int2)(uvX.x, uvY); \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    if (nv_type == 1) \\\n\
    { \\\n\
        UV.s0123456789abcdef = UV.s1032547698badcfe; \\\n\
    } \\\n\
 \\\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8}; \\\n\
    vxc_uchar16 maskShiftUv = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8}; \\\n\
    int4 offsetUV = uvX - uvX.x; \\\n\
 \\\n\
    vxc_ushort8 diffY, diffUV; \\\n\
    _viv_asm(COPY, diffY, sx, 16); \\\n\
    _viv_asm(COPY, diffUV, offsetUV, 16); \\\n\
 \\\n\
    vxc_ushort8 constData = 8; \\\n\
    VXC_DP2x8(maskShift, diffY, constData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
                uniCalculateYShift_2x8); \\\n\
    VXC_DP2x8(maskShiftUv, diffUV, constData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                uniCalculateUVShift_2x8); \\\n\
    VXC_BitExtract(Y, Y, Y, maskShift, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_BitExtract(UV, UV, UV, maskShiftUv, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_short8 tmpY; \\\n\
    vxc_char16 tmpUV; \\\n\
    short tmpVal = 16; \\\n\
    VXC_DP2x8(tmpY, Y, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertYtoShortSub16_2x8); \\\n\
    tmpVal = 128; \\\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8); \\\n\
 \\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    VXC_DP4x4(DstB_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
NV12_OPT_SH_IMPL(U8toU8,  vxc_uchar8, int4,  vxc_uchar8, 8)\n\
NV12_OPT_SH_IMPL(U8toI8,  vxc_char8,  int4,  vxc_char8,  8)\n\
NV12_OPT_SH_IMPL(U8toI16, vxc_short8, int4,  vxc_short8, 16)\n\
NV12_OPT_SH_IMPL(U8toF16, vxc_half8,  half4, vxc_short8, 16)\n\
\n\
#define NV12_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_nv12_scale_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t uv_img, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             nv_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    uint4 gidx = get_global_id(0); \\\n\
    uint gidy = get_global_id(1); \\\n\
    gidx += (uint4)(0, 1, 2, 3); \\\n\
 \\\n\
    uint dy = (gidy * yrIntFloat_16) >> 16; \\\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16; \\\n\
    int sy = convert_int(dy) + (*yOffset); \\\n\
    int4 sx = convert_int4(dx) + (*xOffset); \\\n\
    int4 uvX = sx & 0xfffffffe; \\\n\
    int uvY = sy >> 1; \\\n\
 \\\n\
    vxc_uchar16 Y, UV; \\\n\
    int2 coord = (int2)(sx.x, sy); \\\n\
    int2 coord_uv = (int2)(uvX.x, uvY); \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x = sx.y; \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x = sx.z; \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x = sx.w; \\\n\
    VXC_ReadImage(Y, y_img, coord, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_uv.x = uvX.y; \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_uv.x = uvX.z; \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_uv.x = uvX.w; \\\n\
    VXC_ReadImage(UV, uv_img,coord_uv, 0,VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    if (nv_type == 1) \\\n\
    { \\\n\
        UV.s01234567 = UV.s10325476; \\\n\
    } \\\n\
 \\\n\
    vxc_short8 tmpY; \\\n\
    vxc_char16 tmpUV; \\\n\
    short tmpVal = 16; \\\n\
    VXC_DP2x8(tmpY, Y, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertYtoShortSub16_2x8); \\\n\
    tmpVal = 128; \\\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8); \\\n\
 \\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    VXC_DP4x4(DstB_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertNV12toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
NV12_SH_IMPL(U8toU8,  vxc_uchar8, int4,  vxc_uchar8, 8)\n\
NV12_SH_IMPL(U8toI8,  vxc_char8,  int4,  vxc_char8,  8)\n\
NV12_SH_IMPL(U8toI16, vxc_short8, int4,  vxc_short8, 16)\n\
NV12_SH_IMPL(U8toF16, vxc_half8,  half4, vxc_short8, 16)\n\
"; /* end of pre_process_nv12_scale_vx*/

static const char pre_process_rgb_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
_viv_uniform VXC_512Bits uniUnpackToR;\n\
_viv_uniform VXC_512Bits uniUnpackToG;\n\
_viv_uniform VXC_512Bits uniUnpackToB;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float outputZP;\n\
_viv_uniform int r_order;\n\
_viv_uniform int b_order;\n\
\n\
#define DESCALE(x) (((x) + (1<<19)) >> 20)\n\
\n\
#define IMAGE_PRE_PROCESS(dst_name, conv_type, dst_type, copy_type) \\\n\
__kernel void pre_process_rgb_scale_U8to##dst_name \\\n\
    ( \\\n\
__read_only image2d_array_t  input, \\\n\
__write_only image2d_array_t output, \\\n\
        global int           *xRatio, \\\n\
        global int           *yRatio, \\\n\
        global int           *xOffset, \\\n\
        global int           *yOffset, \\\n\
               float         rMean, \\\n\
               float         gMean, \\\n\
               float         bMean, \\\n\
               float         r_scale, \\\n\
               int           reverse_channel, \\\n\
               int           trans, \\\n\
               float         g_scale, \\\n\
               float         b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos       = get_global_id(0); \\\n\
    int yPos        = get_global_id(1); \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    /*x*/ \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
    /*y*/ \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0RGB1, line0RGB2; \\\n\
    vxc_uchar16 line1RGB3, line1RGB4; \\\n\
    int4 coord; \\\n\
    sx = (sx + (*xOffset)) * 3; \\\n\
    coord.xyz    = sx.xyz; \\\n\
    coord.w        = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0RGB1, input, coord.xw, 0, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0RGB1, input, coord.yw, 0, VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0RGB2, input, coord.zw, 0, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0RGB2, input, coord1, 0, VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1RGB3, input, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1RGB3, input, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1RGB4, input, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1RGB4, input, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 bgrMean = (float4)(bMean, gMean, rMean, 0); \\\n\
 \\\n\
    bgrMean *= (float4)(b_scale, g_scale, r_scale, 0); \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 tt; \\\n\
    vxc_uchar4 val; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, r_order, 0); \\\n\
 \\\n\
    vxc_uchar8 line1, line2; \\\n\
 \\\n\
    /*R*/ \\\n\
    VXC_DP2x8(line1, line0RGB1, line0RGB2, \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToR); \\\n\
    VXC_DP2x8(line2, line1RGB3, line1RGB4, \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToR); \\\n\
 \\\n\
    VXC_DP4x4(test01, line1, line1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line1, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line2, line2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line2, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    /*convert U8 to dst*/ \\\n\
    dst_type dst; \\\n\
    tmp_dst = tmp_dst * r_scale - bgrMean.zzzz; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    conv_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    copy_type result; \\\n\
    _viv_asm(COPY, result, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, result, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    /*G*/ \\\n\
    VXC_DP2x8(line1, line0RGB1, line0RGB2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToG); \\\n\
    VXC_DP2x8(line2, line1RGB3, line1RGB4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToG); \\\n\
 \\\n\
    coord_out.z = 1; \\\n\
    VXC_DP4x4(test01, line1, line1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line1, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line2, line2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line2, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
            uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * g_scale - bgrMean.y; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, result, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, result, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    /*B*/ \\\n\
    VXC_DP2x8(line1, line0RGB1, line0RGB2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToB); \\\n\
    VXC_DP2x8(line2, line1RGB3, line1RGB4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToB); \\\n\
 \\\n\
    coord_out.z = b_order; \\\n\
    VXC_DP4x4(test01, line1, line1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line1, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line2, line2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line2, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * b_scale - bgrMean.x; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, result, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, result, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS(U8,  uint4, vxc_uchar16, vxc_uchar16)\n\
IMAGE_PRE_PROCESS(I8,  int4,  vxc_char16,  vxc_char16)\n\
IMAGE_PRE_PROCESS(I16, int4,  vxc_short8,  vxc_short8)\n\
IMAGE_PRE_PROCESS(F16, half4, vxc_half8,   vxc_short8)\n\
"; /* end of pre_process_rgb_vx*/

static const char pre_process_rgb888_planar_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int4 rgb_order;\n\
\n\
#define RESIZE_BILINEAR_4X1(scale, mean, output, _coord) \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.w; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    coord_in.x = coord.x; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * scale * output_scale - mean * scale * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, _coord, dst, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define PRE_PROCESS_RGB888_PLANAR_16BITS(dst_name, dst_type, conv_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_scale_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
 \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    int4 coord_in = (int4)(0, 0, 0, 0); \\\n\
    sx = sx + *xOffset; \\\n\
    coord = sx.xyzw; \\\n\
    coord_in.y = sy + *yOffset; \\\n\
    coord_in.x = coord.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.w; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    coord_in.x = coord.x; \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 tt; \\\n\
    vxc_uchar4 val; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, yPos, yPos); \\\n\
    coord_out.yzw += rgb_order.xyz; \\\n\
 \\\n\
    vxc_uchar8 line1, line2; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    conv_type dst0; \\\n\
    dst_type dst1; \\\n\
    copy_type dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    RESIZE_BILINEAR_4X1(g_scale, gMean, output, coord_out.xz) \\\n\
    RESIZE_BILINEAR_4X1(b_scale, bMean, output, coord_out.xw) \\\n\
}\n\
PRE_PROCESS_RGB888_PLANAR_16BITS(F16, vxc_half8,  half4, vxc_short8)\n\
PRE_PROCESS_RGB888_PLANAR_16BITS(I16, vxc_short8, int4,  vxc_short8)\n\
\n\
#define PRE_PROCESS_RGB888_PLANAR_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_scale_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos  = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz = sx.xyz; \\\n\
    coord.w   = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    int4 coord_in = (int4)(coord.xw, 0, 0); \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord1.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, yPos, yPos); \\\n\
    coord_out.yzw += rgb_order.xyz; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    int4 dst0; \\\n\
    write_type dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_in.x = coord.x; \\\n\
    coord_in.z = 1; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord1.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * g_scale * output_scale - gMean * g_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out.xz, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_in.x = coord.x; \\\n\
    coord_in.z = 2; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord1.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * b_scale * output_scale - bMean * b_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out.xw, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_RGB888_PLANAR_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_RGB888_PLANAR_8BITS(I8, vxc_char16)"; /* end of pre_process_rgb888_planar_0_vx*/

static const char pre_process_rgb888_planar_1_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataMeanStddevLo_2x8;\n\
_viv_uniform VXC_512Bits uniDataMeanStddevHi_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int4 rgb_order;\n\
\n\
#define PRE_PROCESS_RGB888_PLANAR_COPY_16BITS(dst_name, dst_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2)(*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    dst_type dst0, dst1; \\\n\
 \\\n\
    int4 coord_in = (int4)(coord.xy, 0, 0); \\\n\
    VXC_ReadImage2DArray(src0, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src1, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src2, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord.x = coord.z + 8; \\\n\
    float4 paramData0 = (float4)(rMean * r_scale * output_scale - output_zp,\\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    copy_type tmp_dst; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst1, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    _viv_asm(COPY, tmp_dst, dst0, 16); \\\n\
    int4 coord_out = coord; \\\n\
    coord_out.yw = coord_out.ww + rgb_order.xy; \\\n\
    VXC_WriteImage(output, coord_out.zy, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmp_dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_out.xy, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData1 = (float4)(gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
    VXC_DP2x8(dst0, src1, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    _viv_asm(COPY, tmp_dst, dst0, 16); \\\n\
    VXC_WriteImage(output, coord_out.zw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmp_dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_out.xw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
    VXC_DP2x8(dst0, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst1, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    _viv_asm(COPY, tmp_dst, dst0, 16); \\\n\
    coord_out.w = coord.w + rgb_order.z; \\\n\
    VXC_WriteImage(output, coord_out.zw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmp_dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_out.xw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_RGB888_PLANAR_COPY_16BITS(F16, vxc_half8,  vxc_short8)\n\
PRE_PROCESS_RGB888_PLANAR_COPY_16BITS(I16, vxc_short8, vxc_short8)\n\
\n\
#define PRE_PROCESS_GRAY_COPY_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2) (*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    write_type dst; \\\n\
 \\\n\
    int4 coord_in = (int4)(coord.xy, 0, 0); \\\n\
    VXC_ReadImage2DArray(src0, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src1, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src2, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord; \\\n\
    coord_out.xyw = coord.www + rgb_order.xyz; \\\n\
    float4 paramData0 = (float4)(rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zx, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData1 = (float4)(gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
 \\\n\
    VXC_DP2x8(dst, src1, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src1, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zy, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
 \\\n\
    VXC_DP2x8(dst, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src2, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zw, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_GRAY_COPY_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb888_planar_1_vx*/

static const char pre_process_rgb888_planar_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l00_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l10_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l01_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l11_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l21_4x4;\n\
_viv_uniform int4 rgb_order;\n\
\n\
__kernel void pre_process_rgb888_planar_4over3_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
          global int             *xRatio,\n\
          global int             *yRatio,\n\
          global int             *xOffset,\n\
          global int             *yOffset,\n\
                 float           rMean,\n\
                 float           gMean,\n\
                 float           bMean,\n\
                 float           r_scale,\n\
                 int             reverse,\n\
                 int             height,\n\
                 float           g_scale,\n\
                 float           b_scale\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_out;\n\
\n\
    vxc_uchar16 src0, src1, src2, src3;\n\
    vxc_uchar16 dst0, dst1, dst2;\n\
\n\
    VXC_ReadImage2DArray(src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src2, input, coord_in, VXC_5BITOFFSET_XY(0, 2),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src3, input, coord_in, VXC_5BITOFFSET_XY(0, 3),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z ++;\n\
    coord_out.xy = (coord_in.xy >> 2) * 3;\n\
    coord_out.zw = coord_in.yy + (int2)(1, 2);\n\
\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    int4 coord_r = coord_out;\n\
    coord_r.yzw += rgb_order.xxx;\n\
    VXC_WriteImage(output, coord_r.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_r.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_r.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage2DArray(src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src2, input, coord_in, VXC_5BITOFFSET_XY(0, 2),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src3, input, coord_in, VXC_5BITOFFSET_XY(0, 3),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z ++;\n\
\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    int4 coord_g = coord_out;\n\
    coord_g.yzw += rgb_order.yyy;\n\
    VXC_WriteImage(output, coord_g.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_g.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_g.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage2DArray(src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src2, input, coord_in, VXC_5BITOFFSET_XY(0, 2),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src3, input, coord_in, VXC_5BITOFFSET_XY(0, 3),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    int4 coord_b = coord_out;\n\
    coord_b.yzw += rgb_order.zzz;\n\
    VXC_WriteImage(output, coord_b.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_b.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_b.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_rgb888_planar_half_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
          global int             *xRatio,\n\
          global int             *yRatio,\n\
          global int             *xOffset,\n\
          global int             *yOffset,\n\
                 float           rMean,\n\
                 float           gMean,\n\
                 float           bMean,\n\
                 float           r_scale,\n\
                 int             reverse,\n\
                 int             height,\n\
                 float           g_scale,\n\
                 float           b_scale\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_uchar16 src0, src1, src2;\n\
\n\
    VXC_ReadImage2DArray(src0, input, coord_in, 0,\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z ++;\n\
    VXC_ReadImage2DArray(src1, input, coord_in, 0,\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z ++;\n\
    VXC_ReadImage2DArray(src2, input, coord_in, 0,\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = coord_in.xy >> 1;\n\
\n\
    int4 coord_rgb = coord.xyyy;\n\
    coord_rgb.yzw += rgb_order.xyz;\n\
    VXC_WriteImage(output, coord_rgb.xy, src0.s02468ace, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_rgb.xz, src1.s02468ace, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_rgb.xw, src2.s02468ace, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_rgb888_planar_2_vx*/

static const char pre_process_rgb888_planar_nhwc_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_1_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define RESIZE_BILINEAR_4X1(scale, mean) \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.w; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    coord_in.x = coord.x; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * scale * output_scale - mean * scale * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst);\n\
#define PRE_PROCESS_RGB888_PLANAR_16BITS(dst_name, dst_type, conv_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_scale_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
 \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    int4 coord_in = (int4)(0, 0, 0, 0); \\\n\
    sx = sx + *xOffset; \\\n\
    coord = sx.xyzw; \\\n\
    coord_in.y = sy + *yOffset; \\\n\
    coord_in.x = coord.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.w; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    coord_in.x = coord.x; \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 tt; \\\n\
    vxc_uchar4 val; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, yPos, yPos); \\\n\
    coord_out.x = coord_out.x * 3; \\\n\
    coord_out.z = coord_out.x + 8; \\\n\
 \\\n\
    vxc_uchar8 line1, line2; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    conv_type dst0; \\\n\
    dst_type dst1, dst2; \\\n\
    copy_type data0, data1, dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    RESIZE_BILINEAR_4X1(g_scale, gMean) \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    RESIZE_BILINEAR_4X1(b_scale, bMean) \\\n\
    VXC_DP2x8(dst2, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, data0, dst1, 16); \\\n\
    _viv_asm(COPY, data1, dst2, 16); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_0_2x8); \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_1_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_RGB888_PLANAR_16BITS(F16, vxc_half8,  half4, vxc_short8)\n\
PRE_PROCESS_RGB888_PLANAR_16BITS(I16, vxc_short8, int4,  vxc_short8)\n\
\n\
#define PRE_PROCESS_RGB888_PLANAR_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_scale_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos  = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz = sx.xyz; \\\n\
    coord.w   = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    int4 coord_in = (int4)(coord.xw, 0, 0); \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord1.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int2 coord_out = (int2)(xPos.x, yPos); \\\n\
    coord_out.x = coord_out.x * 3; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    int4 dst0; \\\n\
    write_type dst1, dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    coord_in.x = coord.x; \\\n\
    coord_in.z = 1; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord1.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * g_scale * output_scale - gMean * g_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    coord_in.x = coord.x; \\\n\
    coord_in.z = 2; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.y; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord.z; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = coord1.x; \\\n\
    VXC_ReadImage2DArray(line0Y, input, coord_in, 0, \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(line1Y, input, coord_in, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * b_scale * output_scale - bMean * b_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    VXC_DP2x8(dst, dst1, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_0_2x8); \\\n\
    VXC_DP2x8(dst, dst1, dst1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_1_2x8); \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_RGB888_PLANAR_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_RGB888_PLANAR_8BITS(I8, vxc_char16)"; /* end of pre_process_rgb888_planar_nhwc_0_vx*/

static const char pre_process_rgb888_planar_nhwc_1_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataMeanStddevLo_2x8;\n\
_viv_uniform VXC_512Bits uniDataMeanStddevHi_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_2_2x8;\n\
\n\
#define PRE_PROCESS_RGB888_PLANAR_COPY_16BITS(dst_name, dst_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_copy_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2)(*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    dst_type dst0, dst1; \\\n\
 \\\n\
    int4 coord_in = (int4)(coord.xy, 0, 0); \\\n\
    VXC_ReadImage2DArray(src0, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src1, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src2, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord; \\\n\
    coord_out.z = coord_out.z * 3; \\\n\
    coord_out.x = coord_out.z + 8; \\\n\
    float4 paramData0 = (float4)(rMean * output_scale * r_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    copy_type data0, data1, data2, dst; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    float4 paramData1 = (float4)(gMean * output_scale * g_scale - output_zp,\\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
    VXC_DP2x8(dst0, src1, paramData_f16, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    _viv_asm(COPY, data0, dst0, 16); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * output_scale * b_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
    VXC_DP2x8(dst1, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    _viv_asm(COPY, data1, dst1, 16); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_0_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_1_2x8); \\\n\
    VXC_WriteImage(output, coord_out.xw, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_RGB888_PLANAR_COPY_16BITS(F16, vxc_half8,  vxc_short8)\n\
PRE_PROCESS_RGB888_PLANAR_COPY_16BITS(I16, vxc_short8, vxc_short8)\n\
\n\
#define PRE_PROCESS_GRAY_COPY_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_copy_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2) (*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    write_type dst0, dst1, dst2, dst3; \\\n\
 \\\n\
    int4 coord_in = (int4)(coord.xy, 0, 0); \\\n\
    VXC_ReadImage2DArray(src0, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src1, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.z ++; \\\n\
    VXC_ReadImage2DArray(src2, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord; \\\n\
    coord_out.z = coord_out.z * 3; \\\n\
    coord_out.x = coord_out.z + 16; \\\n\
    float4 paramData0 = (float4)(rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
 \\\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
 \\\n\
    float4 paramData1 = (float4)(gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
 \\\n\
    VXC_DP2x8(dst0, src1, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
 \\\n\
    VXC_DP2x8(dst1, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_0_2x8); \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_1_2x8); \\\n\
    VXC_DP2x8(dst3, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_2_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zw, dst2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output, coord_out.xw, dst3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_GRAY_COPY_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb888_planar_nhwc_1_vx*/

static const char pre_process_rgb888_planar_nhwc_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_2_2x8;\n\
\n\
__kernel void pre_process_rgb888_planar_half_U8toU8_nhwc\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
          global int             *xRatio,\n\
          global int             *yRatio,\n\
          global int             *xOffset,\n\
          global int             *yOffset,\n\
                 float           rMean,\n\
                 float           gMean,\n\
                 float           bMean,\n\
                 float           r_scale,\n\
                 int             reverse,\n\
                 float           g_scale,\n\
                 float           b_scale\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_uchar16 src0, src1, src2;\n\
\n\
    VXC_ReadImage2DArray(src0, input, coord_in, 0,\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z ++;\n\
    VXC_ReadImage2DArray(src1, input, coord_in, 0,\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.z ++;\n\
    VXC_ReadImage2DArray(src2, input, coord_in, 0,\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int4 coord;\n\
    coord.xy = coord_in.xy >> 1;\n\
\n\
    coord.x = coord.x * 3;\n\
    coord.z = coord.x + 16;\n\
\n\
    vxc_uchar16 dst0, dst1;\n\
    src0.lo = src0.s02468ace;\n\
    src0.hi = src1.s02468ace;\n\
    src1.lo = src2.s02468ace;\n\
\n\
    VXC_DP2x8(dst0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uni8BitsDataInterleave_0_2x8);\n\
    VXC_DP2x8(dst0, src0, src1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\n\
        uni8BitsDataInterleave_1_2x8);\n\
    VXC_DP2x8(dst1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uni8BitsDataInterleave_2_2x8);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_rgb888_planar_nhwc_2_vx*/

static const char pre_process_rgb888_planar_sep_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int4 rgb_order;\n\
\n\
#define RESIZE_BILINEAR_4X1(input, scale, mean, output, _coord) \\\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * scale * output_scale - scale * mean * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, _coord, dst, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define RGB888_PLANAR_SEP_16BITS(dst_name, dst_type, conv_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_scale_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
 \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz = sx.xyz; \\\n\
    coord.w = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0Y, input0, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input0, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 tt; \\\n\
    vxc_uchar4 val; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, yPos, yPos); \\\n\
    coord_out.yzw += rgb_order.xyz; \\\n\
 \\\n\
    vxc_uchar8 line1, line2; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    conv_type dst0; \\\n\
    dst_type dst1; \\\n\
    copy_type dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    RESIZE_BILINEAR_4X1(input1, g_scale, gMean, output, coord_out.xz) \\\n\
    RESIZE_BILINEAR_4X1(input2, b_scale, bMean, output, coord_out.xw) \\\n\
}\n\
RGB888_PLANAR_SEP_16BITS(F16, vxc_half8,  half4, vxc_short8)\n\
RGB888_PLANAR_SEP_16BITS(I16, vxc_short8, int4,  vxc_short8)\n\
\n\
#define RGB888_PLANAR_SEP_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_scale_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos  = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz = sx.xyz; \\\n\
    coord.w   = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0Y, input0, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input0, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, yPos, yPos); \\\n\
    coord_out.yzw += rgb_order.xyz; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    int4 dst0; \\\n\
    write_type dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line0Y, input1, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input1, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input1, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input1, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input1, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input1, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input1, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input1, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * g_scale * output_scale - gMean * g_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out.xz, \\\n\
        dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line0Y, input2, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input2, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input2, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input2, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input2, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input2, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input2, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input2, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * b_scale * output_scale - bMean * b_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out.xw, \\\n\
        dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RGB888_PLANAR_SEP_8BITS(U8, vxc_uchar16)\n\
RGB888_PLANAR_SEP_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb888_planar_sep_0_vx*/

static const char pre_process_rgb888_planar_sep_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataMeanStddevLo_2x8;\n\
_viv_uniform VXC_512Bits uniDataMeanStddevHi_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int4 rgb_order;\n\
\n\
#define RGB888_PLANAR_SEP_COPY_16BITS(dst_name, dst_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2)(*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    dst_type dst0, dst1; \\\n\
 \\\n\
    VXC_ReadImage(src0, input0, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input1, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input2, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord.x = coord.z + 8; \\\n\
    float4 paramData0 = (float4)(rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    copy_type tmp_dst; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst1, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    _viv_asm(COPY, tmp_dst, dst0, 16); \\\n\
    int4 coord_out = coord; \\\n\
    coord_out.yw = coord_out.ww + rgb_order.xy; \\\n\
    VXC_WriteImage(output, coord_out.zy, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmp_dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_out.xy, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData1 = (float4)(gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
    VXC_DP2x8(dst0, src1, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst1, src1, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    _viv_asm(COPY, tmp_dst, dst0, 16); \\\n\
    VXC_WriteImage(output, coord_out.zw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmp_dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_out.xw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
    VXC_DP2x8(dst0, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst1, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    _viv_asm(COPY, tmp_dst, dst0, 16); \\\n\
    coord_out.w = coord.w + rgb_order.z; \\\n\
    VXC_WriteImage(output, coord_out.zw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmp_dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_out.xw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RGB888_PLANAR_SEP_COPY_16BITS(F16, vxc_half8,  vxc_short8)\n\
RGB888_PLANAR_SEP_COPY_16BITS(I16, vxc_short8, vxc_short8)\n\
\n\
#define PRE_PROCESS_GRAY_COPY_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 int             height, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2) (*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    write_type dst; \\\n\
 \\\n\
    VXC_ReadImage(src0, input0, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input1, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input2, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord.wwzw; \\\n\
    coord_out.xyw += rgb_order.xyz; \\\n\
    float4 paramData0 = (float4)(rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zx, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData1 = (float4)(gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
 \\\n\
    VXC_DP2x8(dst, src1, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src1, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zy, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
 \\\n\
    VXC_DP2x8(dst, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src2, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zw, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_GRAY_COPY_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb888_planar_sep_1_vx*/

static const char pre_process_rgb888_planar_sep_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l00_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l10_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l01_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l11_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l21_4x4;\n\
_viv_uniform int4 rgb_order;\n\
\n\
__kernel void pre_process_rgb888_planar_sep_4over3_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_array_t input1,\n\
    __read_only  image2d_array_t input2,\n\
    __write_only image2d_array_t output,\n\
          global int             *xRatio,\n\
          global int             *yRatio,\n\
          global int             *xOffset,\n\
          global int             *yOffset,\n\
                 float           rMean,\n\
                 float           gMean,\n\
                 float           bMean,\n\
                 float           r_scale,\n\
                 int             reverse,\n\
                 int             height,\n\
                 float           g_scale,\n\
                 float           b_scale\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
    int4 coord_out;\n\
\n\
    vxc_uchar16 src0, src1, src2, src3;\n\
    vxc_uchar16 dst0, dst1, dst2;\n\
\n\
    VXC_ReadImage(src0, input0, coord_in, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input0, coord_in, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input0, coord_in, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src3, input0, coord_in, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.xy = (coord_in.xy >> 2) * 3;\n\
    coord_out.zw = coord_in.yy + (int2)(1, 2);\n\
\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    int4 coord_r = coord_out;\n\
    coord_r.yzw += rgb_order.xxx;\n\
    VXC_WriteImage(output, coord_r.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_r.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_r.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(src0, input1, coord_in, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord_in, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input1, coord_in, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src3, input1, coord_in, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    int4 coord_g = coord_out;\n\
    coord_g.yzw += rgb_order.yyy;\n\
    VXC_WriteImage(output, coord_g.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_g.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_g.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(src0, input2, coord_in, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input2, coord_in, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input2, coord_in, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src3, input2, coord_in, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    int4 coord_b = coord_out;\n\
    coord_b.yzw += rgb_order.zzz;\n\
    VXC_WriteImage(output, coord_b.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_b.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_b.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_rgb888_planar_sep_half_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_array_t input1,\n\
    __read_only  image2d_array_t input2,\n\
    __write_only image2d_array_t output,\n\
          global int             *xRatio,\n\
          global int             *yRatio,\n\
          global int             *xOffset,\n\
          global int             *yOffset,\n\
                 float           rMean,\n\
                 float           gMean,\n\
                 float           bMean,\n\
                 float           r_scale,\n\
                 int             reverse,\n\
                 int             height,\n\
                 float           g_scale,\n\
                 float           b_scale\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, src1, src2;\n\
\n\
    VXC_ReadImage(src0, input0, coord_in.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord_in.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input2, coord_in.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.zw = coord_in.xy >> 1;\n\
\n\
    int4 coord_rgb = coord_in.zwww;\n\
    coord_rgb.yzw += rgb_order.xyz;\n\
    VXC_WriteImage(output, coord_rgb.xy, src0.s02468ace, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_rgb.xz, src1.s02468ace, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_rgb.xw, src2.s02468ace, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_rgb888_planar_sep_2_vx*/

static const char pre_process_rgb888_planar_sep_nhwc_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_1_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define RESIZE_BILINEAR_4X1(input, scale, mean) \\\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * scale * output_scale - mean * scale * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst);\n\
\n\
#define RGB888_PLANAR_SEP_16BITS(dst_name, dst_type, conv_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_scale_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
 \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz = sx.xyz; \\\n\
    coord.w = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0Y, input0, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input0, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 tt; \\\n\
    vxc_uchar4 val; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, yPos, yPos); \\\n\
    coord_out.x = coord_out.x * 3; \\\n\
    coord_out.z = coord_out.x + 8; \\\n\
 \\\n\
    vxc_uchar8 line1, line2; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    conv_type dst0; \\\n\
    dst_type dst1, dst2; \\\n\
    copy_type data0, data1, dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    _viv_asm(CONV, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    RESIZE_BILINEAR_4X1(input1, g_scale, gMean) \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    RESIZE_BILINEAR_4X1(input2, b_scale, bMean) \\\n\
    VXC_DP2x8(dst2, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, data0, dst1, 16); \\\n\
    _viv_asm(COPY, data1, dst2, 16); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_0_2x8); \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_1_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RGB888_PLANAR_SEP_16BITS(F16, vxc_half8,  half4, vxc_short8)\n\
RGB888_PLANAR_SEP_16BITS(I16, vxc_short8, int4,  vxc_short8)\n\
\n\
#define RGB888_PLANAR_SEP_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_scale_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos = get_global_id(0); \\\n\
    int yPos  = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz = sx.xyz; \\\n\
    coord.w   = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0Y, input0, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input0, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input0, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input0, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int2 coord_out = (int2)(xPos.x, yPos); \\\n\
    coord_out.x = coord_out.x * 3; \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    int4 dst0; \\\n\
    write_type dst1, dst; \\\n\
    tmp_dst = tmp_dst * r_scale * output_scale - rMean * r_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_ReadImage(line0Y, input1, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input1, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input1, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input1, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input1, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input1, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input1, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input1, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * g_scale * output_scale - gMean * g_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
 \\\n\
    VXC_ReadImage(line0Y, input2, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input2, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input2, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input2, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input2, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input2, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input2, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input2, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
    tmp_dst = tmp_dst * b_scale * output_scale - bMean * b_scale * output_scale + output_zp; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtract8Data_2x8); \\\n\
    VXC_DP2x8(dst, dst1, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_0_2x8); \\\n\
    VXC_DP2x8(dst, dst1, dst1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_1_2x8); \\\n\
    VXC_WriteImage(output, coord_out.xy, dst, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RGB888_PLANAR_SEP_8BITS(U8, vxc_uchar16)\n\
RGB888_PLANAR_SEP_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb888_planar_sep_nhwc_0_vx*/

static const char pre_process_rgb888_planar_sep_nhwc_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataMeanStddevLo_2x8;\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni16BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_2_2x8;\n\
\n\
#define RGB888_PLANAR_SEP_COPY_16BITS(dst_name, dst_type, copy_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_copy_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2)(*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    dst_type dst0, dst1; \\\n\
 \\\n\
    VXC_ReadImage(src0, input0, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input1, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input2, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord; \\\n\
    coord_out.z = coord_out.z * 3; \\\n\
    coord_out.x = coord_out.z + 8; \\\n\
    float4 paramData0 = (float4)(rMean * r_scale * output_scale - output_zp,\\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    copy_type data0, data1, data2, dst; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    float4 paramData1 = (float4)(gMean * g_scale * output_scale - output_zp,\\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
    VXC_DP2x8(dst0, src1, paramData_f16, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    _viv_asm(COPY, data0, dst0, 16); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * b_scale * output_scale - output_zp,\\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
    VXC_DP2x8(dst0, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    _viv_asm(COPY, data1, dst0, 16); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_0_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni16BitsDataInterleave_1_2x8); \\\n\
    VXC_WriteImage(output, coord_out.xw, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RGB888_PLANAR_SEP_COPY_16BITS(F16, vxc_half8,  vxc_short8)\n\
RGB888_PLANAR_SEP_COPY_16BITS(I16, vxc_short8, vxc_short8)\n\
\n\
#define PRE_PROCESS_GRAY_COPY_8BITS(dst_name, write_type) \\\n\
__kernel void pre_process_rgb888_planar_sep_copy_U8to##dst_name##_nhwc \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int             *xRatio, \\\n\
          global int             *yRatio, \\\n\
          global int             *xOffset, \\\n\
          global int             *yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2) (*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    write_type dst0, dst1, dst2, dst3; \\\n\
 \\\n\
    VXC_ReadImage(src0, input0, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input1, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input2, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord; \\\n\
    coord_out.z = coord_out.z * 3; \\\n\
    coord_out.x = coord_out.z + 16; \\\n\
    float4 paramData0 = (float4)(rMean * r_scale * output_scale - output_zp,\\\n\
        rMean * r_scale * output_scale - output_zp, \\\n\
        rMean * r_scale * output_scale - output_zp, r_scale * output_scale); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData0); \\\n\
 \\\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
 \\\n\
    float4 paramData1 = (float4)(gMean * g_scale * output_scale - output_zp,\\\n\
        gMean * g_scale * output_scale - output_zp, \\\n\
        gMean * g_scale * output_scale - output_zp, g_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData1); \\\n\
 \\\n\
    VXC_DP2x8(dst0, src1, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
 \\\n\
    float4 paramData2 = (float4)(bMean * b_scale * output_scale - output_zp,\\\n\
        bMean * b_scale * output_scale - output_zp, \\\n\
        bMean * b_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    _viv_asm(CONV, paramData_f16, paramData2); \\\n\
 \\\n\
    VXC_DP2x8(dst1, src2, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_0_2x8); \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_1_2x8); \\\n\
    VXC_DP2x8(dst3, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uni8BitsDataInterleave_2_2x8); \\\n\
    VXC_WriteImage(output, coord_out.zw, dst2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output, coord_out.xw, dst3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
PRE_PROCESS_GRAY_COPY_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb888_planar_sep_nhwc_1_vx*/

static const char pre_process_rgb888_planar_sep_nhwc_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_0_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_1_2x8;\n\
_viv_uniform VXC_512Bits uni8BitsDataInterleave_2_2x8;\n\
\n\
__kernel void pre_process_rgb888_planar_sep_half_U8toU8_nhwc\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_array_t input1,\n\
    __read_only  image2d_array_t input2,\n\
    __write_only image2d_array_t output,\n\
          global int             *xRatio,\n\
          global int             *yRatio,\n\
          global int             *xOffset,\n\
          global int             *yOffset,\n\
                 float           rMean,\n\
                 float           gMean,\n\
                 float           bMean,\n\
                 float           r_scale,\n\
                 int             reverse,\n\
                 float           g_scale,\n\
                 float           b_scale\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, src1, src2;\n\
\n\
    VXC_ReadImage(src0, input0, coord_in.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord_in.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input2, coord_in.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int4 coord;\n\
    coord.xy = coord_in.xy >> 1;\n\
\n\
    coord.x = coord.x * 3;\n\
    coord.z = coord.x + 16;\n\
\n\
    vxc_uchar16 dst0, dst1;\n\
    src0.lo = src0.s02468ace;\n\
    src0.hi = src1.s02468ace;\n\
    src1.lo = src2.s02468ace;\n\
\n\
    VXC_DP2x8(dst0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uni8BitsDataInterleave_0_2x8);\n\
    VXC_DP2x8(dst0, src0, src1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\n\
        uni8BitsDataInterleave_1_2x8);\n\
    VXC_DP2x8(dst1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uni8BitsDataInterleave_2_2x8);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_rgb888_planar_sep_nhwc_2_vx*/

static const char pre_process_rgb_copy_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform int r_order;\n\
_viv_uniform int b_order;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part3_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part3_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part3_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float4 param_data;\n\
_viv_uniform float4 rgb_scale;\n\
\n\
#define IMAGE_PRE_PROCESS_COPY_16BITS(dst_name, dst_type, copy_type, convert_type) \\\n\
__kernel void pre_process_rgb_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
         global int              *xRatio, \\\n\
         global int              *yRatio, \\\n\
         global int              *xOffset, \\\n\
         global int              *yOffset, \\\n\
                float            rMean, \\\n\
                float            gMean, \\\n\
                float            bMean, \\\n\
                float            r_scale, \\\n\
                int              reverse_channel, \\\n\
                int              trans, \\\n\
                float            g_scale, \\\n\
                float            b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 coord      = (int2)(get_global_id(0) * 3, get_global_id(1)); \\\n\
 \\\n\
    coord.xy = coord.xy + (int2) (*xOffset * 3 + 16, *yOffset); \\\n\
    vxc_uchar16 src0, src1; \\\n\
    dst_type   dst0; \\\n\
    copy_type   dst; \\\n\
 \\\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(-16, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), r_order, 0); \\\n\
    float4 tmp0, tmp1; \\\n\
    convert_type result0, result1; \\\n\
 \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.x - param_data.x; \\\n\
    tmp1 = tmp1 * rgb_scale.x - param_data.x; \\\n\
    _viv_asm(CONV_RTE, result0, tmp0); \\\n\
    _viv_asm(CONV_RTE, result1, tmp1); \\\n\
    VXC_DP2x8(dst0, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = 1; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.y - param_data.y; \\\n\
    tmp1 = tmp1 * rgb_scale.y - param_data.y; \\\n\
    _viv_asm(CONV_RTE, result0, tmp0); \\\n\
    _viv_asm(CONV_RTE, result1, tmp1); \\\n\
    VXC_DP2x8(dst0, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = b_order; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.z - param_data.z; \\\n\
    tmp1 = tmp1 * rgb_scale.z - param_data.z; \\\n\
    _viv_asm(CONV_RTE, result0, tmp0); \\\n\
    _viv_asm(CONV_RTE, result1, tmp1); \\\n\
    VXC_DP2x8(dst0, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS_COPY_16BITS(I16, vxc_short8, vxc_short8, int4)\n\
IMAGE_PRE_PROCESS_COPY_16BITS(F16, vxc_half8,  vxc_short8, half4)\n\
\n\
#define IMAGE_PRE_PROCESS_COPY_8BITS(dst_name, dst_type) \\\n\
__kernel void pre_process_rgb_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
         global int              *xRatio, \\\n\
         global int              *yRatio, \\\n\
         global int              *xOffset, \\\n\
         global int              *yOffset, \\\n\
                float            rMean, \\\n\
                float            gMean, \\\n\
                float            bMean, \\\n\
                float            r_scale, \\\n\
                int              reverse_channel, \\\n\
                int              trans, \\\n\
                float            g_scale, \\\n\
                float            b_scale \\\n\
    ) \\\n\
{ \\\n\
    int2 coord      = (int2)(get_global_id(0) * 3, get_global_id(1)); \\\n\
    coord.xy = coord.xy + (int2) (*xOffset * 3 + 16, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    dst_type dst; \\\n\
 \\\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(-16, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
    VXC_ReadImage(src2, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), r_order, 0); \\\n\
    float4 tmp0, tmp1; \\\n\
    int4 result0, result1; \\\n\
 \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.x - param_data.x; \\\n\
    tmp1 = tmp1 * rgb_scale.x - param_data.x; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_DP4x4(tmp0, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part2_4x4); \\\n\
    VXC_DP4x4(tmp1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part3_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.x - param_data.x; \\\n\
    tmp1 = tmp1 * rgb_scale.x - param_data.x; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = 1; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.y - param_data.y; \\\n\
    tmp1 = tmp1 * rgb_scale.y - param_data.y; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_DP4x4(tmp0, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part2_4x4); \\\n\
    VXC_DP4x4(tmp1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part3_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.y - param_data.y; \\\n\
    tmp1 = tmp1 * rgb_scale.y - param_data.y; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = b_order; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.z - param_data.z; \\\n\
    tmp1 = tmp1 * rgb_scale.z - param_data.z; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_DP4x4(tmp0, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part2_4x4); \\\n\
    VXC_DP4x4(tmp1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part3_4x4); \\\n\
    tmp0 = tmp0 * rgb_scale.z - param_data.z; \\\n\
    tmp1 = tmp1 * rgb_scale.z - param_data.z; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS_COPY_8BITS(U8, vxc_uchar16)\n\
IMAGE_PRE_PROCESS_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb_copy_vx*/

static const char pre_process_yuv420_copy_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpB1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoR_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiR_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float output_scale;\n\
\n\
#define YUV420_COPY_SH_IMPL(name, dst_type) \\\n\
__kernel void pre_process_yuv420_copy_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t u_img, \\\n\
    __read_only  image2d_array_t v_img, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int *           xRatio, \\\n\
          global int *           yRatio, \\\n\
          global int *           xOffset, \\\n\
          global int *           yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 pos = (int4)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset), 0, 0); \\\n\
    int2 pos1 = (int2)((get_global_id(0) + (*xOffset)) >> 1, (get_global_id(1) + (*yOffset)) >> 1); \\\n\
    vxc_uchar16 Y; \\\n\
    vxc_uchar8 U, V; \\\n\
    vxc_int4 C0, C1, C2, C3; \\\n\
    vxc_uchar16 R, G, B; \\\n\
    dst_type dst0, dst1, dst2; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, pos.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, pos1.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, pos1.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    /*C = Y - 16;*/ \\\n\
    /*D = U - 128;*/ \\\n\
    /*E = V - 128;*/ \\\n\
    /* calculate R*/ \\\n\
    /* ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]*/ \\\n\
    int tmpV = -56992; \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4); \\\n\
 \\\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
 \\\n\
    /* calculate G*/ \\\n\
    /* ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]*/ \\\n\
    /* 298Y - 208V*/ \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4); \\\n\
    /* 34784 - 100U*/ \\\n\
    ushort tmpG = 34784; \\\n\
    vxc_ushort8 tmpDstG; \\\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8); \\\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4); \\\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4); \\\n\
    VXC_DP4x4(G, C2, tmpDstG, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG3rd_4x4); \\\n\
    VXC_DP4x4(G, C3, tmpDstG, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG4th_4x4); \\\n\
 \\\n\
    /* calculate B*/ \\\n\
    /* ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]*/ \\\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4); \\\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4); \\\n\
    tmpV = -70688; \\\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
 \\\n\
    float4  paramData = (float4)(bMean * b_scale * output_scale - output_zp,\\\n\
                                 gMean * g_scale * output_scale - output_zp, \\\n\
                                 rMean * r_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
 \\\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8); \\\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8); \\\n\
 \\\n\
    paramData.w = g_scale * output_scale; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
 \\\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8); \\\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8); \\\n\
 \\\n\
    paramData.w = r_scale * output_scale; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
 \\\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8); \\\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8); \\\n\
 \\\n\
    pos = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    pos.z = bOrder; \\\n\
    VXC_WriteImage2DArray(output, pos, dst0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    pos.z = 1; \\\n\
    VXC_WriteImage2DArray(output, pos, dst1, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    pos.z = rOrder; \\\n\
    VXC_WriteImage2DArray(output, pos, dst2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
YUV420_COPY_SH_IMPL(U8toU8, vxc_uchar16)\n\
YUV420_COPY_SH_IMPL(U8toI8, vxc_char16)\n\
\n\
#define YUV420_COPY_16BITS_SH_IMPL(name, dst_type) \\\n\
__kernel void pre_process_yuv420_copy_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t u_img, \\\n\
    __read_only  image2d_array_t v_img, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int *           xRatio, \\\n\
          global int *           yRatio, \\\n\
          global int *           xOffset, \\\n\
          global int *           yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 pos = (int4)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset), 0, 0); \\\n\
    int2 pos1 = (int2)((get_global_id(0) + (*xOffset)) >> 1, (get_global_id(1) + (*yOffset)) >> 1); \\\n\
    vxc_uchar16 Y; \\\n\
    vxc_uchar8 U, V; \\\n\
    vxc_int4 C0, C1, C2, C3; \\\n\
    vxc_uchar16 R, G, B; \\\n\
    dst_type dst0, dst1, dst2, dst3, dst4, dst5; \\\n\
    vxc_short8 out0, out1, out2, out3, out4, out5; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, pos.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, pos1.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, pos1.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int tmpV = -56992; \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4); \\\n\
 \\\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4); \\\n\
 \\\n\
    ushort tmpG = 34784; \\\n\
    vxc_ushort8 tmpDstG; \\\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8); \\\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4); \\\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4); \\\n\
    VXC_DP4x4(G, C2, tmpDstG, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG3rd_4x4); \\\n\
    VXC_DP4x4(G, C3, tmpDstG, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG4th_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4); \\\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4); \\\n\
    tmpV = -70688; \\\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
 \\\n\
    float4  paramData = (float4)(bMean * b_scale * output_scale - output_zp, \\\n\
                                 gMean * g_scale * output_scale - output_zp, \\\n\
                                 rMean * r_scale * output_scale - output_zp, b_scale * output_scale); \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
 \\\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8); \\\n\
    VXC_DP2x8(dst1, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8); \\\n\
 \\\n\
    paramData.w = g_scale * output_scale; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
    VXC_DP2x8(dst2, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8); \\\n\
    VXC_DP2x8(dst3, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8); \\\n\
 \\\n\
    paramData.w = r_scale * output_scale; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
    VXC_DP2x8(dst4, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8); \\\n\
    VXC_DP2x8(dst5, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8); \\\n\
 \\\n\
    _viv_asm(COPY, out0, dst0, 16); \\\n\
    _viv_asm(COPY, out1, dst1, 16); \\\n\
    _viv_asm(COPY, out2, dst2, 16); \\\n\
    _viv_asm(COPY, out3, dst3, 16); \\\n\
    _viv_asm(COPY, out4, dst4, 16); \\\n\
    _viv_asm(COPY, out5, dst5, 16); \\\n\
 \\\n\
    pos = (int4)(get_global_id(0), get_global_id(1), bOrder, get_global_id(0) + 8); \\\n\
    VXC_WriteImage2DArray(output, pos.xyzz, out0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, pos.wyzz, out1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    pos.z = 1; \\\n\
    VXC_WriteImage2DArray(output, pos.xyzz, out2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, pos.wyzz, out3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    pos.z = rOrder; \\\n\
    VXC_WriteImage2DArray(output, pos.xyzz, out4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, pos.wyzz, out5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
YUV420_COPY_16BITS_SH_IMPL(U8toF16, vxc_half8)\n\
YUV420_COPY_16BITS_SH_IMPL(U8toI16, vxc_short8)\n\
"; /* end of pre_process_yuv420_copy_vx*/

static const char pre_process_yuv420_scale_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float output_scale;\n\
\n\
#define YUV420_SCALE_8BITS_SH_IMPL(name, dst_type) \\\n\
__kernel void pre_process_yuv420_scale_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t u_img, \\\n\
    __read_only  image2d_array_t v_img, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int *           xRatio, \\\n\
          global int *           yRatio, \\\n\
          global int *           xOffset, \\\n\
          global int *           yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    gidx += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14); \\\n\
    int4 sx = fx & 0xffff8000; \\\n\
    int fy, sy; \\\n\
    fx -= sx; \\\n\
    sx = sx >> 15; \\\n\
    fx = (fx +(1 << 4)) >> 5; \\\n\
 \\\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14); \\\n\
    sy = fy & 0xffff8000; \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    sy = sy < 0 ? 0 : sy; \\\n\
    fy = fy < 0 ? 0 : fy; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
    sx += (*xOffset); \\\n\
    sy += (*yOffset); \\\n\
    int4 srcPos = (int4)(sx.x, sy, get_global_id(2), 0); \\\n\
    int4 srcPos1 = (int4)(sx.x >> 1, sy >> 1, get_global_id(2), 0); \\\n\
    int4 srcPos2 = (int4)(sx.x >> 1, (sy + 1) >> 1, get_global_id(2), 0); \\\n\
 \\\n\
    vxc_uchar16 Y, U, V; \\\n\
    vxc_int4 C0, C1, C2, C3; \\\n\
    vxc_uchar16 R, G, B; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.x + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.x + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.y; \\\n\
    srcPos1.x = sx.y >> 1; \\\n\
    srcPos2.x = sx.y >> 1; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.y + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.y + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.z; \\\n\
    srcPos1.x = sx.z >> 1; \\\n\
    srcPos2.x = sx.z >> 1; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.z + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.z + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.w; \\\n\
    srcPos1.x = sx.w >> 1; \\\n\
    srcPos2.x = sx.w >> 1; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.w + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.w + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int tmpV = -56992; \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4); \\\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4); \\\n\
 \\\n\
    ushort tmpG = 34784; \\\n\
    vxc_ushort8 tmpDstG, tmpDstG1; \\\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8); \\\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8); \\\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4); \\\n\
    tmpV = -70688; \\\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
 \\\n\
    int4 result, temp1, temp2; \\\n\
    int4 tmpData0, tmpData1; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
 \\\n\
    tmpV = 1 << 19; \\\n\
    dst_type dst; \\\n\
    float4 tmpDst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - bMean) * b_scale; \\\n\
    dstPos.z = bOrder; \\\n\
    result = convert_int4_rte(tmpDst * output_scale + output_zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - gMean) * g_scale; \\\n\
    dstPos.z = 1; \\\n\
    result = convert_int4_rte(tmpDst * output_scale + output_zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - rMean) * r_scale; \\\n\
    dstPos.z = rOrder; \\\n\
    result = convert_int4_rte(tmpDst * output_scale + output_zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
YUV420_SCALE_8BITS_SH_IMPL(U8toU8, vxc_uchar8)\n\
YUV420_SCALE_8BITS_SH_IMPL(U8toI8, vxc_char8)\n\
"; /* end of pre_process_yuv420_scale_0_vx*/

static const char pre_process_yuv420_scale_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define YUV420_SCALE_16BITS_SH_IMPL(name, dst_type, conv_type) \\\n\
__kernel void pre_process_yuv420_scale_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t y_img, \\\n\
    __read_only  image2d_array_t u_img, \\\n\
    __read_only  image2d_array_t v_img, \\\n\
    __write_only image2d_array_t output, \\\n\
          global int *           xRatio, \\\n\
          global int *           yRatio, \\\n\
          global int *           xOffset, \\\n\
          global int *           yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    gidx += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14); \\\n\
    int4 sx = fx & 0xffff8000; \\\n\
    int fy, sy; \\\n\
    fx -= sx; \\\n\
    sx = sx >> 15; \\\n\
    fx = (fx +(1 << 4)) >> 5; \\\n\
 \\\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14); \\\n\
    sy = fy & 0xffff8000; \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    sy = sy < 0 ? 0 : sy; \\\n\
    fy = fy < 0 ? 0 : fy; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
    sx += (*xOffset); \\\n\
    sy += (*yOffset); \\\n\
    int4 srcPos = (int4)(sx.x, sy, get_global_id(2), 0); \\\n\
    int4 srcPos1 = (int4)(sx.x >> 1, sy >> 1, get_global_id(2), 0); \\\n\
    int4 srcPos2 = (int4)(sx.x >> 1, (sy + 1) >> 1, get_global_id(2), 0); \\\n\
 \\\n\
    vxc_uchar16 Y, U, V; \\\n\
    vxc_int4 C0, C1, C2, C3; \\\n\
    vxc_uchar16 R, G, B; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.x + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.x + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.y; \\\n\
    srcPos1.x = sx.y >> 1; \\\n\
    srcPos2.x = sx.y >> 1; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.y + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.y + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.z; \\\n\
    srcPos1.x = sx.z >> 1; \\\n\
    srcPos2.x = sx.z >> 1; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.z + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.z + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.w; \\\n\
    srcPos1.x = sx.w >> 1; \\\n\
    srcPos2.x = sx.w >> 1; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, 0, VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos1.x = (sx.w + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos1, 0, VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos1, 0, VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0)); \\\n\
    srcPos2.x = (sx.w + 1) >> 1; \\\n\
    VXC_ReadImage(U, u_img, srcPos2, 0, VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos2, 0, VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int tmpV = -56992; \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4); \\\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4); \\\n\
 \\\n\
    ushort tmpG = 34784; \\\n\
    vxc_ushort8 tmpDstG, tmpDstG1; \\\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8); \\\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8); \\\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4); \\\n\
    tmpV = -70688; \\\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
 \\\n\
    int4 result, temp1, temp2; \\\n\
    int4 tmpData0, tmpData1; \\\n\
    dst_type tmpResult; \\\n\
    conv_type tmpVal; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
 \\\n\
    tmpV = 1 << 19; \\\n\
    vxc_short8 dst; \\\n\
    float4 tmpDst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - bMean) * b_scale; \\\n\
    dstPos.z = bOrder; \\\n\
    tmpDst = tmpDst * output_scale + output_zp; \\\n\
    _viv_asm(CONV_RTE, tmpVal, tmpDst); \\\n\
    VXC_DP2x8(tmpResult, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmpResult, 8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - gMean) * g_scale; \\\n\
    dstPos.z = 1; \\\n\
    tmpDst = tmpDst * output_scale + output_zp; \\\n\
    _viv_asm(CONV_RTE, tmpVal, tmpDst); \\\n\
    VXC_DP2x8(tmpResult, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmpResult, 8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - rMean) * r_scale; \\\n\
    dstPos.z = rOrder; \\\n\
    tmpDst = tmpDst * output_scale + output_zp; \\\n\
    _viv_asm(CONV_RTE, tmpVal, tmpDst); \\\n\
    VXC_DP2x8(tmpResult, tmpVal, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmpResult, 8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
YUV420_SCALE_16BITS_SH_IMPL(U8toF16, vxc_half8,  half4)\n\
YUV420_SCALE_16BITS_SH_IMPL(U8toI16, vxc_short8, int4)\n\
"; /* end of pre_process_yuv420_scale_1_vx*/

static const char pre_process_yuv422_copy_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar_b;\n\
_viv_uniform float outputScaleVar_g;\n\
_viv_uniform float outputScaleVar_r;\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform VXC_512Bits uniConvertYUV422toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertYUV422toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertYUV422toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniExtractYUVtoShortSub_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUchartoFp32_4x4;\n\
\n\
#define YUV422_COPY_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_yuv422_copy_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             yuv422_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int sy = gidy + (*yOffset); \\\n\
    int sx = gidx * 2 + (*xOffset * 2); \\\n\
 \\\n\
    vxc_uchar8 YUV; \\\n\
    vxc_short8 tmpYUV; \\\n\
 \\\n\
    VXC_ReadImage(YUV, input, (int2)(sx,sy), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    if (yuv422_type == 1) \\\n\
    { \\\n\
        YUV.s01234567 = YUV.s10325476; \\\n\
    } \\\n\
 \\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    vxc_short2 value = (vxc_short2)(128,16); \\\n\
    VXC_DP2x8(tmpYUV, YUV, value, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractYUVtoShortSub_2x8); \\\n\
    VXC_DP4x4(DstB_uchar, tmpYUV, tmpYUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\\\n\
                                          uniConvertYUV422toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpYUV, tmpYUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\\\n\
                                          uniConvertYUV422toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpYUV, tmpYUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\\\n\
                                          uniConvertYUV422toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
              VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(gidx, gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
YUV422_COPY_SH_IMPL(U8toU8,  vxc_uchar4, int4,  vxc_uchar4, 4)\n\
YUV422_COPY_SH_IMPL(U8toI8,  vxc_char4,  int4,  vxc_char4,  4)\n\
YUV422_COPY_SH_IMPL(U8toI16, vxc_short4, int4,  vxc_short4, 8)\n\
YUV422_COPY_SH_IMPL(U8toF16, vxc_half4,  half4, vxc_short4, 8)\n\
"; /* end of pre_process_yuv422_copy_vx*/

static const char pre_process_yuv422_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar_b;\n\
_viv_uniform float outputScaleVar_g;\n\
_viv_uniform float outputScaleVar_r;\n\
\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform uint  xrIntFloat_16;\n\
_viv_uniform uint  yrIntFloat_16;\n\
\n\
_viv_uniform VXC_512Bits uniConvertYUV422toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertYUV422toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertYUV422toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniExtractUVtoCharSub128_2x8;\n\
_viv_uniform VXC_512Bits uniExtractYtoShortSub16_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUchartoFp32_4x4;\n\
\n\
#define uyvy422 1\n\
\n\
#define YUV422_SH_IMPL(name, dst_type, conv_type, save_type, copy_bytes) \\\n\
__kernel void pre_process_yuv422_scale_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    global       int*            xRatio, \\\n\
    global       int*            yRatio, \\\n\
    global       int*            xOffset, \\\n\
    global       int*            yOffset, \\\n\
                 float           rMean, \\\n\
                 float           gMean, \\\n\
                 float           bMean, \\\n\
                 float           r_scale, \\\n\
                 int             reverse_channel, \\\n\
                 int             trans, \\\n\
                 int             yuv422_type, \\\n\
                 float           g_scale, \\\n\
                 float           b_scale \\\n\
    ) \\\n\
{ \\\n\
    int4 gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    gidx += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    uint dy = (convert_uint(gidy) * yrIntFloat_16) >> 16; \\\n\
    uint4 dx = (convert_uint4(gidx) * xrIntFloat_16) >> 16; \\\n\
    int sy = convert_int(dy) + (*yOffset); \\\n\
    int4 sx = (convert_int4(dx)+ *xOffset) * 2; \\\n\
 \\\n\
    vxc_uchar4 Y; \\\n\
    vxc_uchar8 UV; \\\n\
    vxc_short4 tmpY; \\\n\
    vxc_char8 tmpUV; \\\n\
    short tmpVal = 16; \\\n\
    int y_offset = 0; \\\n\
    int u_offset = 1; \\\n\
    int v_offset = 3; \\\n\
\\\n\
    if (yuv422_type == uyvy422) \\\n\
    { \\\n\
        y_offset = 1; \\\n\
        u_offset = 0; \\\n\
        v_offset = 2; \\\n\
    } \\\n\
\\\n\
    int4 coord_Y = (int4)(sx.x + y_offset, sy, 0, 0); \\\n\
    int4 coord_U = (int4)((sx.x >> 2) * 4 + u_offset, sy, 0, 0); \\\n\
    int4 coord_V = (int4)((sx.x >> 2) * 4 + v_offset, sy, 0, 0); \\\n\
\\\n\
    VXC_ReadImage2DArray(Y, input, coord_Y, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_Y.x = sx.y + y_offset; \\\n\
    VXC_ReadImage2DArray(Y, input, coord_Y, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_Y.x = sx.z + y_offset; \\\n\
    VXC_ReadImage2DArray(Y, input, coord_Y, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_Y.x = sx.w + y_offset; \\\n\
    VXC_ReadImage2DArray(Y, input, coord_Y, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    sx = (sx >> 2) * 4 + u_offset; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_U, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_U.x = sx.y; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_U, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_U.x = sx.z; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_U, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_U.x = sx.w; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_U, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
\\\n\
    sx = sx - u_offset + v_offset; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_V, 0, VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_V.x = sx.y; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_V, 0, VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_V.x = sx.z; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_V, 0, VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_V.x = sx.w; \\\n\
    VXC_ReadImage2DArray(UV, input, coord_V, 0, VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP4x4(tmpY, Y, tmpVal, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractYtoShortSub16_4x4); \\\n\
    tmpVal = 128; \\\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractUVtoCharSub128_2x8); \\\n\
    vxc_uchar4 dst_test; \\\n\
    VXC_DP2x8(dst_test, dx, dx, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniExtract8Data_2x8); \\\n\
\\\n\
    float4 tmpDstB, tmpDstG, tmpDstR; \\\n\
    vxc_uchar4 DstB_uchar, DstG_uchar, DstR_uchar; \\\n\
    VXC_DP4x4(DstB_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\\\n\
                                       uniConvertYUV422toB_4x4); \\\n\
    VXC_DP4x4(DstG_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\\\n\
                                       uniConvertYUV422toG_4x4); \\\n\
    VXC_DP4x4(DstR_uchar, tmpY, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\\\n\
                                       uniConvertYUV422toR_4x4); \\\n\
    VXC_DP4x4(tmpDstB, DstB_uchar, DstB_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstG, DstG_uchar, DstG_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
    VXC_DP4x4(tmpDstR, DstR_uchar, DstR_uchar, \\\n\
                       VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUchartoFp32_4x4); \\\n\
 \\\n\
    conv_type result; \\\n\
    dst_type dst0; \\\n\
    save_type dst; \\\n\
    int4 dstPos = (int4)(gidx.x, gidy, 0, 0); \\\n\
    tmpDstB = tmpDstB * outputScaleVar_b + bMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstB); \\\n\
    dstPos.z = bOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstG = tmpDstG * outputScaleVar_g + gMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstG); \\\n\
    dstPos.z = 1; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    tmpDstR = tmpDstR * outputScaleVar_r + rMeanScaleVarZp; \\\n\
    _viv_asm(CONV_RTE, result, tmpDstR); \\\n\
    dstPos.z = rOrder; \\\n\
    VXC_DP2x8(dst0, result, result, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, copy_bytes); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
YUV422_SH_IMPL(U8toU8,  vxc_uchar4, int4,  vxc_uchar4, 4)\n\
YUV422_SH_IMPL(U8toI8,  vxc_char4,  int4,  vxc_char4,  4)\n\
YUV422_SH_IMPL(U8toI16, vxc_short4, int4,  vxc_short4, 8)\n\
YUV422_SH_IMPL(U8toF16, vxc_half4,  half4, vxc_short4, 8)\n\
"; /* end of pre_process_yuv422_scale_vx*/

static const char pre_process_yuv444_copy_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpB1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoR_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiR_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_yuv444_copy_U8toU8(\n\
    __read_only image2d_t            y_img,\n\
    __read_only image2d_t            u_img,\n\
    __read_only image2d_t            v_img,\n\
    __write_only image2d_array_t    output,\n\
        global int *                xRatio,\n\
        global int *                yRatio,\n\
        global int *               xOffset,\n\
        global int *               yOffset,\n\
               float                 rMean,\n\
               float                 gMean,\n\
               float                 bMean,\n\
               float               r_scale,\n\
               int         reverse_channel,\n\
               int                   trans,\n\
               float               g_scale,\n\
               float               b_scale\n\
    )\n\
{\n\
    int2 pos = (int2)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset));\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
    vxc_uchar16 dst0, dst1, dst2;\n\
\n\
    VXC_ReadImage(Y, y_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16;\n\
    //D = U - 128;\n\
    //E = V - 128;\n\
    // calculate R\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4);\n\
\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // calculate G\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG0, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG0, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2_2x8);\n\
\n\
    VXC_DP4x4(G, C0, tmpDstG0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
\n\
    // calculate B\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    float4  paramData = (float4)(bMean * b_scale * outputScale - zp, gMean * g_scale * outputScale - zp,\\\n\
        rMean * r_scale * outputScale - zp, b_scale * outputScale);\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8);\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8);\n\
\n\
    paramData.w = g_scale * outputScale;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8);\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8);\n\
\n\
    paramData.w = r_scale * outputScale;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8);\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8);\n\
\n\
    int4 pos1 = (int4)(get_global_id(0), get_global_id(1), bOrder, 0);\n\
    VXC_WriteImage2DArray(output, pos1, dst0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = 1;\n\
    VXC_WriteImage2DArray(output, pos1, dst1, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = rOrder;\n\
    VXC_WriteImage2DArray(output, pos1, dst2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_yuv444_copy_U8toF16(\n\
    __read_only image2d_t            y_img,\n\
    __read_only image2d_t            u_img,\n\
    __read_only image2d_t            v_img,\n\
    __write_only image2d_array_t    output,\n\
        global int *                xRatio,\n\
        global int *                yRatio,\n\
        global int *               xOffset,\n\
        global int *               yOffset,\n\
               float                 rMean,\n\
               float                 gMean,\n\
               float                 bMean,\n\
               float               r_scale,\n\
               int         reverse_channel,\n\
               int                   trans,\n\
               float               g_scale,\n\
               float               b_scale\n\
    )\n\
{\n\
    int2 pos = (int2)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset));\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
    vxc_half8 dst0, dst1, dst2, dst3, dst4, dst5;\n\
    vxc_short8 out0, out1, out2, out3, out4, out5;\n\
\n\
    VXC_ReadImage(Y, y_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16;\n\
    //D = U - 128;\n\
    //E = V - 128;\n\
    // calculate R\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4);\n\
\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // calculate G\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG0, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG0, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2_2x8);\n\
\n\
    VXC_DP4x4(G, C0, tmpDstG0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
\n\
    // calculate B\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    float4  paramData = (float4)(bMean * b_scale * outputScale, gMean * g_scale * outputScale,\\\n\
        rMean * r_scale * outputScale, b_scale * outputScale);\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8);\n\
    VXC_DP2x8(dst1, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8);\n\
\n\
    paramData.w = g_scale * outputScale;\n\
     _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst2, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8);\n\
    VXC_DP2x8(dst3, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8);\n\
\n\
    paramData.w = r_scale * outputScale;\n\
     _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst4, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8);\n\
    VXC_DP2x8(dst5, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8);\n\
\n\
    _viv_asm(COPY, out0, dst0, 16);\n\
    _viv_asm(COPY, out1, dst1, 16);\n\
    _viv_asm(COPY, out2, dst2, 16);\n\
    _viv_asm(COPY, out3, dst3, 16);\n\
    _viv_asm(COPY, out4, dst4, 16);\n\
    _viv_asm(COPY, out5, dst5, 16);\n\
\n\
    int4 pos1 = (int4)(get_global_id(0), get_global_id(1), bOrder, get_global_id(0) + 8);\n\
    VXC_WriteImage2DArray(output, pos1.xyzz, out0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos1.wyzz, out1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = 1;\n\
    VXC_WriteImage2DArray(output, pos1.xyzz, out2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos1.wyzz, out3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = rOrder;\n\
    VXC_WriteImage2DArray(output, pos1.xyzz, out4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos1.wyzz, out5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_yuv444_copy_u8_vx*/

static const char pre_process_yuv444_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
#define IMAGE_PRE_PROCESS_YUV444_QINT(dst_name, dst_type) \\\n\
__kernel void pre_process_yuv444_scale_U8to##dst_name( \\\n\
    __read_only image2d_t y_img, __read_only image2d_t u_img, \\\n\
    __read_only image2d_t v_img, __write_only image2d_array_t    output, \\\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset, \\\n\
    float rMean, float gMean, float bMean, float r_scale, int reverse_channel, int trans, \\\n\
    float g_scale, float b_scale) \\\n\
{ \\\n\
    int4 gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    gidx += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14); \\\n\
    int4 sx = fx & 0xffff8000;  \\\n\
    int fy, sy; \\\n\
    fx -= sx; \\\n\
    sx = sx >> 15; \\\n\
    fx = (fx +(1 << 4)) >> 5; \\\n\
 \\\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14); \\\n\
    sy = fy & 0xffff8000;  \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    sy = sy < 0 ? 0 : sy; \\\n\
    fy = fy < 0 ? 0 : fy; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
    sx += (*xOffset); \\\n\
    sy += (*yOffset); \\\n\
    int2 srcPos = (int2)(sx.x, sy); \\\n\
 \\\n\
    vxc_uchar16 Y, U, V; \\\n\
    vxc_int4 C0, C1, C2, C3; \\\n\
    vxc_uchar16 R, G, B; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.y; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.z; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.w; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int tmpV = -56992; \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4); \\\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4); \\\n\
 \\\n\
    ushort tmpG = 34784; \\\n\
    vxc_ushort8 tmpDstG, tmpDstG1; \\\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8); \\\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8); \\\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4); \\\n\
    tmpV = -70688; \\\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
 \\\n\
    int4 result, temp1, temp2; \\\n\
    int4 tmpData0, tmpData1; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
 \\\n\
    tmpV = 1 << 19; \\\n\
    dst_type dst; \\\n\
    float4 tmpDst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - bMean) * b_scale; \\\n\
    dstPos.z = bOrder; \\\n\
    result = convert_int4_rte(tmpDst * outputScale + zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - gMean) * g_scale; \\\n\
    dstPos.z = 1; \\\n\
    result = convert_int4_rte(tmpDst * outputScale + zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - rMean) * r_scale; \\\n\
    dstPos.z = rOrder; \\\n\
    result = convert_int4_rte(tmpDst * outputScale + zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS_YUV444_QINT(U8, vxc_uchar8)\n\
IMAGE_PRE_PROCESS_YUV444_QINT(I8, vxc_char8)\n\
IMAGE_PRE_PROCESS_YUV444_QINT(I16, vxc_short8)"; /* end of pre_process_yuv444_scale_vx*/

static const char pre_process_yuv444_scale_fp16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
__kernel void pre_process_yuv444_scale_U8toF16(\n\
    __read_only image2d_t y_img, __read_only image2d_t u_img,\n\
    __read_only image2d_t v_img, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float r_scale, int reverse_channel, int trans,\n\
    float g_scale, float b_scale)\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx += (*xOffset);\n\
    sy += (*yOffset);\n\
    int2 srcPos = (int2)(sx.x, sy);\n\
\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.y;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.w;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16; D = U - 128; E = V - 128;\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4);\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    int4 result, temp1, temp2;\n\
    int4 tmpData0, tmpData1;\n\
\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    // temp2 - temp1\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
\n\
    vxc_half8 tmpVal;\n\
    half4 hDst;\n\
    tmpV = 1 << 19;\n\
    vxc_short8 dst;\n\
    float4 tmpDst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - bMean) * b_scale;\n\
    dstPos.z = bOrder;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - gMean) * g_scale;\n\
    dstPos.z = 1;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - rMean) * r_scale;\n\
    dstPos.z = rOrder;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_yuv444_scale_fp16_vx*/

static const char prelu_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#if (VX_VERSION==2)\n\
_viv_uniform VXC_512Bits uniPreluDFPLo_2x8b;\n\
_viv_uniform VXC_512Bits uniPreluDFPHi_2x8b;\n\
__kernel void prelu_I8F16toI8_2D_OPT\n\
(\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_char16 in, dst;\n\
    vxc_char32 src;\n\
    vxc_short8 a0, a1;\n\
    vxc_half8 c0, c1;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a1, param, coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    _viv_asm(COPY, c1, a1, 4);\n\
    src.hi = max(in, 0);\n\
    src.lo = min(in, 0);\n\
\n\
    VXC_DP2x8_b(dst, src.hi, src.lo, c0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluDFPLo_2x8b);\n\
    VXC_DP2x8_b(dst, src.hi, src.lo, c1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniPreluDFPHi_2x8b);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void prelu_I16F16toI16_2D_OPT\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 in, dst;\n\
    vxc_short16 src;\n\
    vxc_short8 a0;\n\
    vxc_half8 c0;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    src.hi = max(in, 0);\n\
    src.lo = min(in, 0);\n\
    VXC_DP2x8_b(dst, src.hi, src.lo, c0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluDFPLo_2x8b);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
#else\n\
_viv_uniform VXC_512Bits uniPreluInt8_2x8;\n\
_viv_uniform VXC_512Bits uniPreluInt16_part0_4x4;\n\
_viv_uniform VXC_512Bits uniPreluInt16_part1_4x4;\n\
__kernel void prelu_I8F16toI8_2D_OPT\n\
(\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 in, dst;\n\
    vxc_char16 src0, src1, src;\n\
    vxc_short8 a0, a1;\n\
    vxc_half8  c0, c1;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a1, param, coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    _viv_asm(COPY, c1, a1, 4);\n\
    src0 = max(in, 0);\n\
    src1 = min(in, 0);\n\
    _viv_asm(COPY, src, src0, 16);\n\
    src.s89abcdef = src1.s01234567;\n\
    VXC_DP2x8(dst, src, c0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluInt8_2x8);\n\
    _viv_asm(COPY, src, src1, 16);\n\
    src.s01234567 = src0.s89abcdef;\n\
    VXC_DP2x8(dst, src, c1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniPreluInt8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void prelu_I16F16toI16_2D_OPT\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_short8 in, dst;\n\
    vxc_short8 src0, src1, src;\n\
    vxc_short8 a0;\n\
    vxc_half8  c0;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    src0 = max(in, 0);\n\
    src1 = min(in, 0);\n\
    _viv_asm(COPY, src, src0, 16);\n\
    src.s4567 = src1.s0123;\n\
    VXC_DP4x4(dst, src, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniPreluInt16_part0_4x4);\n\
    _viv_asm(COPY, src, src1, 16);\n\
    src.s0123 = src0.s4567;\n\
    VXC_DP4x4(dst, src, c0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluInt16_part1_4x4);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
#endif\n\
\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bin_2x8;\n\
_viv_uniform int input0_zp;\n\
_viv_uniform int input1_zp;\n\
_viv_uniform float input0_scale;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform float output_zp;\n\
#define PRELU_F16_3D(name, input_type0, copy_type0, output_type, convert_type, copy_type) \\\n\
    __kernel void prelu_##name( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\\\n\
    float4 vecA, vecB, vecC, vecD;\\\n\
    input_type0 srcA;\\\n\
    copy_type0  src0;\\\n\
    vxc_short8 srcB;\\\n\
    vxc_half8  src1;\\\n\
    input_type0 zp;\\\n\
    VXC_ReadImage2DArray(srcA, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src0, srcA, 16); \\\n\
    VXC_ReadImage2DArray(srcB, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src1, srcB, 16); \\\n\
    \\\n\
    _viv_asm(COPY, zp, input0_zp, 4);\\\n\
    VXC_DP4x4(vecA, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), \\\n\
        uniDataSubZPtoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), \\\n\
        uniDataSubZPtoFp32Part1_4x4);\\\n\
    VXC_DP4x4(vecC, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part0_4x4);\\\n\
    VXC_DP4x4(vecD, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part1_4x4);\\\n\
    \\\n\
    vecA = vecA * input0_scale;\\\n\
    vecB = vecB * input0_scale;\\\n\
    float4 maxData0 = vecA > 0 ? vecA : 0.0; \\\n\
    float4 maxData1 = vecB > 0 ? vecB : 0.0; \\\n\
    float4 minData0 = vecA < 0 ? vecA : 0.0; \\\n\
    float4 minData1 = vecB < 0 ? vecB : 0.0; \\\n\
    vecA = maxData0 + vecC * minData0 + output_zp;\\\n\
    vecB = maxData1 + vecD * minData1 + output_zp;\\\n\
    convert_type dst0, dst1;\\\n\
    _viv_asm(CONV_RTE, dst0, vecA);\\\n\
    _viv_asm(CONV_RTE, dst1, vecB);\\\n\
    output_type dst2;\\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
    copy_type dst;\\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
}\n\
//        name0, name1, input_type0, copy_type0,  output_type, convert_type, copy_type\n\
PRELU_F16_3D(I8F16toI8,   vxc_char16,  vxc_char16,  vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_3D(I8F16toF16,  vxc_char16,  vxc_char16,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(I16F16toI16, vxc_short8,  vxc_short8,  vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_3D(I16F16toF16, vxc_short8,  vxc_short8,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(U8F16toU8,   vxc_uchar16, vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16)\n\
PRELU_F16_3D(U8F16toF16,  vxc_uchar16, vxc_uchar16, vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(F16F16toF16, vxc_short8,  vxc_half8,   vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(F16F16toI8,  vxc_short8,  vxc_half8,   vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_3D(F16F16toI16, vxc_short8,  vxc_half8,   vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_3D(F16F16toU8,  vxc_short8,  vxc_half8,   vxc_uchar16, int4,  vxc_uchar16)\n\
\n\
#define PRELU_F16_2D(name, input_type0, copy_type0, output_type, convert_type, copy_type) \\\n\
    __kernel void prelu_##name##_2D( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    float4 vecA, vecB, vecC, vecD;\\\n\
    input_type0 srcA;\\\n\
    copy_type0  src0;\\\n\
    vxc_short8 srcB;\\\n\
    vxc_half8  src1;\\\n\
    input_type0 zp;\\\n\
    VXC_ReadImage(srcA, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src0, srcA, 16); \\\n\
    VXC_ReadImage(srcB, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src1, srcB, 16); \\\n\
    \\\n\
    _viv_asm(COPY, zp, input0_zp, 4);\\\n\
    VXC_DP4x4(vecA, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part0_4x4);\\\n\
    VXC_DP4x4(vecB, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part1_4x4);\\\n\
    VXC_DP4x4(vecC, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part0_4x4);\\\n\
    VXC_DP4x4(vecD, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part1_4x4);\\\n\
    \\\n\
    vecA = vecA * input0_scale;\\\n\
    vecB = vecB * input0_scale;\\\n\
    float4 maxData0 = vecA > 0 ? vecA : 0.0; \\\n\
    float4 maxData1 = vecB > 0 ? vecB : 0.0; \\\n\
    float4 minData0 = vecA < 0 ? vecA : 0.0; \\\n\
    float4 minData1 = vecB < 0 ? vecB : 0.0; \\\n\
    vecA = maxData0 + vecC * minData0 + output_zp;\\\n\
    vecB = maxData1 + vecD * minData1 + output_zp;\\\n\
    convert_type dst0, dst1;\\\n\
    _viv_asm(CONV_RTE, dst0, vecA);\\\n\
    _viv_asm(CONV_RTE, dst1, vecB);\\\n\
    output_type dst2;\\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
    copy_type dst;\\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
}\n\
PRELU_F16_2D(I8F16toF16,  vxc_char16,  vxc_char16,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(I8F16toI8,   vxc_char16,  vxc_char16,  vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_2D(I16F16toF16, vxc_short8,  vxc_short8,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(U8F16toU8,   vxc_uchar16, vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16)\n\
PRELU_F16_2D(U8F16toF16,  vxc_uchar16, vxc_uchar16, vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(F16F16toF16, vxc_short8,  vxc_half8,   vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(F16F16toI8,  vxc_short8,  vxc_half8,   vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_2D(F16F16toI16, vxc_short8,  vxc_half8,   vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_2D(I16F16toI16, vxc_short8,  vxc_short8,  vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_2D(F16F16toU8,  vxc_short8,  vxc_half8,   vxc_uchar16, int4,  vxc_uchar16)\n\
\n\
#define PRELU_INTEGER_2D(name, src0_type, src1_type, output_type, convert_type, copy_type) \\\n\
    __kernel void prelu_##name##_2D( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    float4 vecA, vecB, vecC, vecD;\\\n\
    src0_type  src0;\\\n\
    src1_type  src1;\\\n\
    short zp0;\\\n\
    short zp1;\\\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    \\\n\
    _viv_asm(COPY, zp0, input0_zp, 2);\\\n\
    VXC_DP4x4(vecA, src0, zp0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part0_4x4);\\\n\
    VXC_DP4x4(vecB, src0, zp0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part1_4x4);\\\n\
    _viv_asm(COPY, zp1, input1_zp, 4);\\\n\
    VXC_DP4x4(vecC, src1, zp1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part0_4x4);\\\n\
    VXC_DP4x4(vecD, src1, zp1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part1_4x4);\\\n\
    \\\n\
    vecA = vecA * input0_scale;\\\n\
    vecB = vecB * input0_scale;\\\n\
    vecC = vecC * input1_scale;\\\n\
    vecD = vecD * input1_scale;\\\n\
    float4 maxData0 = vecA >= 0 ? vecA : 0.0; \\\n\
    float4 maxData1 = vecB >= 0 ? vecB : 0.0; \\\n\
    float4 minData0 = vecA < 0 ? vecA : 0.0; \\\n\
    float4 minData1 = vecB < 0 ? vecB : 0.0; \\\n\
    vecA = maxData0 + vecC * minData0 + output_zp;\\\n\
    vecB = maxData1 + vecD * minData1 + output_zp;\\\n\
    convert_type dst0, dst1;\\\n\
    _viv_asm(CONV_RTE, dst0, vecA);\\\n\
    _viv_asm(CONV_RTE, dst1, vecB);\\\n\
    output_type dst2;\\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
    copy_type dst;\\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
}\n\
PRELU_INTEGER_2D(U8U8toU8,    vxc_uchar16, vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16)\n\
PRELU_INTEGER_2D(U8U8toF16,   vxc_uchar16, vxc_uchar16, vxc_half8,   half4, vxc_short8)\n\
\n\
\n\
\n\
"; /* end of prelu_vx*/

static const char prelu_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_Part0_4x4;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_Part1_4x4;\n\
_viv_uniform VXC_512Bits uniPackedBF16_2x8;\n\
\n\
#define PRELU_BF16F16TOBF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, para_s16; \\\n\
    vxc_half8 para_f16; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(para_s16, param, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, para_f16, para_s16, 16); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_ushort8 src1, src2; \\\n\
    float4 srcA, srcB; \\\n\
    float4 para0_f32, para1_f32; \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, srcA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
    VXC_DP4x4(para0_f32, para_f16, para_f16, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvF16toF32_Part0_4x4);\\\n\
    VXC_DP4x4(para1_f32, para_f16, para_f16, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvF16toF32_Part1_4x4);\\\n\
    srcA = srcA >= 0 ? srcA : srcA * para0_f32; \\\n\
    srcB = srcB >= 0 ? srcB : srcB * para1_f32; \\\n\
    _viv_asm(COPY, src1, srcA, 16); \\\n\
    _viv_asm(COPY, src2, srcB, 16); \\\n\
    VXC_DP2x8(src1, src1, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackedBF16_2x8); \\\n\
    write_fun(output, coord, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void prelu_BF16F16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    PRELU_BF16F16TOBF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
__kernel void prelu_BF16F16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output,\n\
    int                          axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    PRELU_BF16F16TOBF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
#define PRELU_BF16BF16TOBF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, para_s16; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(para_s16, param, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_ushort8 src1, src2; \\\n\
    float4 srcA, srcB; \\\n\
    float4 para0_f32, para1_f32; \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, srcA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
    VXC_DP2x8(src1, para_s16, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, para0_f32, src1, 16); \\\n\
    VXC_DP2x8(src1, para_s16, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, para1_f32, src1, 16); \\\n\
    srcA = srcA >= 0 ? srcA : srcA * para0_f32; \\\n\
    srcB = srcB >= 0 ? srcB : srcB * para1_f32; \\\n\
    _viv_asm(COPY, src1, srcA, 16); \\\n\
    _viv_asm(COPY, src2, srcB, 16); \\\n\
    VXC_DP2x8(src1, src1, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackedBF16_2x8); \\\n\
    write_fun(output, coord, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void prelu_BF16BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    PRELU_BF16BF16TOBF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
__kernel void prelu_BF16BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    PRELU_BF16BF16TOBF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
"; /* end of prelu_BF16_vx*/

static const char random_multinomial_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int class_max_iter;\n\
\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
_viv_uniform VXC_512Bits uniGetSubData0to3_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubData4to7_4x4;\n\
_viv_uniform int iter;\n\
_viv_uniform int stride;\n\
_viv_uniform float re_rand_max;\n\
\n\
uint4 _philox4x32bumpkey(uint4 key)\n\
{\n\
    uint4 mask = (uint4)((uint)0x9E3779B9, (uint)0xBB67AE85, 0, 0);\n\
    //key.x += ((uint)0x9E3779B9);\n\
    //key.y += ((uint)0xBB67AE85);\n\
    key += mask;\n\
    return key;\n\
}\n\
\n\
uint mulhilo32(uint a, uint b, uint* hip)\n\
{\n\
    uint product = (uint)(a * b);\n\
    *hip = mul_hi(a, b);\n\
    return product;\n\
}\n\
\n\
uint mullo32(uint a, uint b)\n\
{\n\
    return a * b;\n\
}\n\
\n\
uint mulhi32(uint a, uint b)\n\
{\n\
    return mul_hi(a, b);\n\
}\n\
\n\
uint4 _philox4x32round(uint4 ctr, uint4 key)\n\
{\n\
    //uint hi0;\n\
    //uint hi1;\n\
    uint PHILOX_M4x32_0 = ((uint)0xD2511F53);\n\
    uint PHILOX_M4x32_1 = ((uint)0xCD9E8D57);\n\
    uint lo0 = mullo32(PHILOX_M4x32_0, ctr.x);\n\
    uint hi0 = mulhi32(PHILOX_M4x32_0, ctr.x);\n\
    uint lo1 = mullo32(PHILOX_M4x32_1, ctr.z);\n\
    uint hi1 = mulhi32(PHILOX_M4x32_1, ctr.z);\n\
    //uint lo0 = mulhilo32(PHILOX_M4x32_0, ctr.x, &hi0);\n\
    //uint lo1 = mulhilo32(PHILOX_M4x32_1, ctr.z, &hi1);\n\
    uint4 out = (uint4)(hi1^ctr.y^key.x, lo1, hi0^ctr.w^key.y, lo0);\n\
    return out;\n\
}\n\
\n\
uint4 philox4x32_R_10(uint4 ctr, uint4 key)\n\
{\n\
    uint i;\n\
    ctr = _philox4x32round(ctr, key);\n\
    for (i = 1; i < 10; i++)\n\
    {\n\
        key = _philox4x32bumpkey(key);\n\
        ctr = _philox4x32round(ctr, key);\n\
    }\n\
    return ctr;\n\
}\n\
\n\
__kernel void random_seed(\n\
    __read_only  image2d_t  seeds,\n\
    __write_only image2d_t  output)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx << 1, gidy, 0, 0);\n\
\n\
    int width = get_image_width(seeds);\n\
    Image s_img = create_image_from_image2d(seeds, 4);\n\
    __global uint* seeds_ptr = (__global uint*)s_img.ptr;\n\
    seeds_ptr = seeds_ptr + coord.x + coord.y * width;\n\
    uint4 key = vload4(0, seeds_ptr);\n\
\n\
    uint4 ctr = (uint4)(0);\n\
    float4 result = 0;\n\
\n\
    width = get_image_width(output);\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    coord.x = gidx * stride + width * coord.y;\n\
    __global float* output_ptr = (__global float*)o_img.ptr;\n\
    output_ptr += coord.x;\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        ctr = philox4x32_R_10(ctr, key);\n\
        result = convert_float4(ctr) * re_rand_max;\n\
        vstore4(result, i, output_ptr);\n\
    }\n\
}\n\
\n\
#define logE    (1.44269502f)\n\
float4 eltwise_unary_exp(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
// N times of 8\n\
// x dim = 1\n\
__kernel void random_multinomial_cdf_F16\n\
    (\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
\n\
    vxc_half8 maxData, data;\n\
    vxc_short8 src0;\n\
    float4 dst0 = 0, dst1 = 0;\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float tmp = 0;\n\
\n\
    int class_max_stride = get_image_width(input);\n\
    int offset = gidy * class_max_stride;\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    __global float* output_ptr = (__global float*)o_img.ptr;\n\
    __global float* cdfPtr = output_ptr + offset;\n\
\n\
    VXC_ReadImage(maxData, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    for(int i = 1; i < class_max_iter; i++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
        _viv_asm(COPY, data, src0, 16);\n\
\n\
        VXC_VertMax3_Half(maxData, maxData, maxData, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    VXC_HorzMax3_Half(maxData, maxData, VXC_MODIFIER(0, 5, 0,VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(maxData, maxData, maxData, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8);\n\
    VXC_HorzMax3_Half(maxData, maxData, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord.x = 0;\n\
    for(int i = 0; i < class_max_iter; i++)\n\
    {\n\
        float4 val0, val1;\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
        _viv_asm(COPY, data, src0, 16);\n\
        VXC_DP4x4(val0, data, maxData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4);\n\
        VXC_DP4x4(val1, data, maxData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4);\n\
        val0 = eltwise_unary_exp(val0);\n\
        val1 = eltwise_unary_exp(val1);\n\
        val0.x += dst1.w;\n\
        dst0 = (float4)(val0.x, (val0.x + val0.y), dot(val0, (float4)(1, 1, 1, 0)), dot(val0, one));\n\
        val1.x += dst0.w;\n\
        dst0 = (float4)(val1.x, (val1.x + val1.y), dot(val1, (float4)(1, 1, 1, 0)), dot(val1, one));\n\
        vstore4(dst0, 0, cdfPtr);\n\
        vstore4(dst1, 1, cdfPtr);\n\
        cdfPtr += 8;\n\
    }\n\
}\n\
\n\
__kernel void random_multinomial_cdf_F32\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
\n\
    vxc_float4 src0, data;\n\
    float maxData0 = FLT_MIN, maxData1 = FLT_MIN;\n\
    uint4 ctr = (uint4)(0);\n\
    float4 dst = 0;\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
\n\
    int class_max_stride = get_image_width(input);\n\
    float tmp = 0;\n\
    int offset = gidy * class_max_stride;\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    __global float* output_ptr = (__global float*)o_img.ptr;\n\
    __global float* cdfPtr = output_ptr + offset;\n\
\n\
    int width = get_image_width(input);\n\
    Image i_img = create_image_from_image2d(input, 4);\n\
    __global float* input_ptr = (__global float*)i_img.ptr;\n\
    input_ptr = input_ptr + coord.x + coord.y * width;\n\
\n\
    float4 maxVal = vload4(0, input_ptr);\n\
    for(int i = 1; i < class_max_iter; i++)\n\
    {\n\
        src0 = vload4(i, input_ptr);\n\
\n\
        maxVal = maxVal > src0 ? maxVal : src0;\n\
    }\n\
    maxVal.xy = maxVal.xy > maxVal.zw ? maxVal.xy : maxVal.zw;\n\
    maxData0 = maxVal.x > maxVal.y ? maxVal.x : maxVal.y;\n\
\n\
    float4 maxData = (float4)(maxData0, maxData0, maxData0, maxData0);\n\
    for(int i = 0; i < class_max_iter; i++)\n\
    {\n\
        float4 val;\n\
        src0 = vload4(i, input_ptr);\n\
        data = src0 - maxData;\n\
        val = eltwise_unary_exp(data);\n\
        val.x += dst.w;\n\
        dst = (float4)(val.x, (val.x + val.y), dot(val, (float4)(1, 1, 1, 0)), dot(val, one));\n\
        vstore4(dst, i, cdfPtr);\n\
    }\n\
}\n\
\n\
uint upper_bound(float* a, int n, float x)\n\
{\n\
    uint l = 0;\n\
    uint h = n;\n\
    while (l < h) {\n\
        int mid = (l + h) >> 1;\n\
        if (x >= a[mid]) {\n\
            l = mid + 1;\n\
        } else {\n\
            h = mid;\n\
        }\n\
    }\n\
    return l;\n\
}\n\
\n\
// one thread calculate 4\n\
__kernel void random_multinomial\n\
    (\n\
    __read_only image2d_t   randoms,\n\
    __read_only image2d_t   cdfs,\n\
   __write_only image2d_t   output,\n\
                int         class_size\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
\n\
    int class_max_stride = get_image_width(cdfs);\n\
    int offset = gidy * class_max_stride;\n\
    Image cdf_img = create_image_from_image2d(cdfs, 4);\n\
    __global float* cdf_ptr = (__global float*)cdf_img.ptr;\n\
    __global float* cdfPtr = cdf_ptr + offset;\n\
\n\
    int width = get_image_width(randoms);\n\
    offset = coord.x + coord.y * width;\n\
    Image r_img = create_image_from_image2d(randoms, 4);\n\
    __global float* randoms_ptr = (__global float*)r_img.ptr;\n\
    randoms_ptr = randoms_ptr + offset;\n\
\n\
    width = get_image_width(output);\n\
    offset = coord.x + coord.y * width;\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    __global uint* output_ptr = (__global uint*)o_img.ptr;\n\
    output_ptr = output_ptr + offset;\n\
\n\
    float4 ran = vload4(0, randoms_ptr);\n\
    float total = cdfPtr[class_size - 1];\n\
    float4 target = ran * total;\n\
\n\
    uint4 out_class = (uint4)(0);\n\
    out_class.x = upper_bound(cdfPtr, class_size, target.x);\n\
    out_class.y = upper_bound(cdfPtr, class_size, target.y);\n\
    out_class.z = upper_bound(cdfPtr, class_size, target.z);\n\
    out_class.w = upper_bound(cdfPtr, class_size, target.w);\n\
\n\
    vstore4(out_class, 0, output_ptr);\n\
}\n\
\n\
"; /* end of random_multinomial_vx*/

static const char reduceall_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
\n\
_viv_uniform VXC_512Bits  uniS8AddAll_16x1;\n\
\n\
#define REDUCEALL_AXIS0_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    int4 sum_val = 0; \\\n\
    result = ones; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        VXC_DP16x1(sum_val, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniS8AddAll_16x1); \\\n\
        if (sum_val.x != 16) \\\n\
        { \\\n\
            result = zeros; \\\n\
            break; \\\n\
        } \\\n\
        coord.x += 16; \\\n\
    } \\\n\
    while(coord.x < axisSize); \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceall_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS0_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceall_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(0));\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS0_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceall_internal_axis0_vx*/

static const char reduceall_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEALL_AXIS1_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.y++; \\\n\
    while(coord.y < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result & val; \\\n\
        coord.y++; \\\n\
    } \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceall_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS1_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceall_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS1_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of reduceall_internal_axis1_vx*/

static const char reduceall_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEALL_AXIS2_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.z++; \\\n\
    while(coord.z < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result & val; \\\n\
        coord.z++; \\\n\
    } \\\n\
    write_fun(output, coord.xy, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceall_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS2_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceall_internal_axis2_vx*/

static const char reduceany_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
\n\
_viv_uniform VXC_512Bits  uniS8AddAll_16x1;\n\
\n\
#define REDUCEANY_AXIS0_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    int4 sum_val = 0; \\\n\
    result = zeros; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        VXC_DP16x1(sum_val, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniS8AddAll_16x1); \\\n\
        if (sum_val.x != 0) \\\n\
        { \\\n\
            result = ones; \\\n\
            break; \\\n\
        } \\\n\
        coord.x += 16; \\\n\
    } \\\n\
    while(coord.x < axisSize); \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceany_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS0_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceany_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(0));\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS0_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceany_internal_axis0_vx*/

static const char reduceany_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEANY_AXIS1_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.y++; \\\n\
    while(coord.y < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result | val; \\\n\
        coord.y++; \\\n\
    } \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceany_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS1_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceany_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS1_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of reduceany_internal_axis1_vx*/

static const char reduceany_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEANY_AXIS2_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.z++; \\\n\
    while(coord.z < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result | val; \\\n\
        coord.z++; \\\n\
    } \\\n\
    write_fun(output, coord.xy, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceany_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS2_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceany_internal_axis2_vx*/

static const char reducemax_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
\n\
#define REDUCEMAX_PROCESS_AXIS0(read_fun, vert_max_fun, horz_max_fun) \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        vert_max_fun(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        vert_max_fun(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        horz_max_fun(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
        horz_max_fun(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS0_SAVE_SAME(save_type, write_fun) \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, val, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
                      OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    vxc_float4 prob; \\\n\
    dst_type vec1; \\\n\
    save_type dst; \\\n\
    VXC_DP4x4(prob, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    prob = ((prob - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, vec1, prob); \\\n\
    _viv_asm(COPY, dst, vec1, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_AXIS0_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
#define REDUCEMAX_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
              save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMAX_AXIS0_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
\n\
REDUCEMAX_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric,\\\n\
                    1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
REDUCEMAX_AXIS0(I16, I16, vxc_short8, vxc_short8, short4,\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale,\\\n\
input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define REDUCEMAX_AXIS0_SAME_2D(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode,\\\n\
                           OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMAX_AXIS0_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
\n\
REDUCEMAX_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric,\\\n\
                    1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,\n\
                   vxc_short8, CONV, 1, 0, inputScale,\\\n\
                   input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\n\
                   CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\n\
                   CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale,\\\n\
input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
\n\
"; /* end of reducemax_internal_axis0_vx*/

static const char reducemax_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMAX_PROCESS_AXIS1(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define REDUCEMAX_PROCESS_AXIS1_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, max, 16); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_AXIS1_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS1(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMAX_AXIS1_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half)\n\
\n\
\n\
REDUCEMAX_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
\n\
\n\
#define REDUCEMAX_AXIS1_SAME_2D(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS1_2D(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMAX_AXIS1_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half)\n\
\n\
REDUCEMAX_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,\n\
vxc_short8, CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(I16, I16, vxc_short8, vxc_short8,\\\n\
short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16,\\\n\
char4, vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
"; /* end of reducemax_internal_axis1_vx*/

static const char reducemax_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMAX_PROCESS_AXIS2(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
#define REDUCEMAX_PROCESS_AXIS2_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, max, 16); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_AXIS2_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun) \\\n\
__kernel void reducemax_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS2_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS2(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun) \\\n\
__kernel void reducemax_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode, OUT_SCALE,\\\n\
    OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMAX_AXIS2_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half)\n\
\n\
\n\
REDUCEMAX_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
\n\
REDUCEMAX_AXIS2(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
\n\
"; /* end of reducemax_internal_axis2_vx*/

static const char reducemin_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
\n\
#define REDUCEMIN_PROCESS_AXIS0(read_fun, vert_min_fun, horz_min_fun) \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        vert_min_fun(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        vert_min_fun(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16)); \\\n\
    horz_min_fun(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    horz_min_fun(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS0_SAVE_SAME(save_type, write_fun) \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, val, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    vxc_float4 prob; \\\n\
    dst_type vec1; \\\n\
    save_type dst; \\\n\
    VXC_DP4x4(prob, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    prob = ((prob - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, vec1, prob); \\\n\
    _viv_asm(COPY, dst, vec1, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_AXIS0_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
#define REDUCEMIN_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
              save_type, conv_mode, OUT_SCALE, OUT_OFFSET,\\\n\
              IN_SCALE, IN_OFFSET, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMIN_AXIS0_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
\n\
REDUCEMIN_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale,\\\n\
0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale,\\\n\
0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
 CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0,\\\n\
inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0,\\\n\
inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8, CONV,\\\n\
1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
\n\
REDUCEMIN_AXIS0(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric,\\\n\
VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
\n\
#define REDUCEMIN_AXIS0_SAME_2D(src_name, dst_name, src_type, copy_type,\\\n\
save_type, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode, OUT_SCALE,\\\n\
                           OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMIN_AXIS0_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
\n\
REDUCEMIN_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
\n\
REDUCEMIN_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric,\\\n\
VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
\n\
\n\
"; /* end of reducemin_internal_axis0_vx*/

static const char reducemin_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMIN_PROCESS_AXIS1(read_fun, vert_min_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, min, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_min_fun(min, min, min, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define REDUCEMIN_PROCESS_AXIS1_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, min, 16); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_AXIS1_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS1(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMIN_AXIS1_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half)\n\
\n\
\n\
REDUCEMIN_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
\n\
\n\
#define REDUCEMIN_AXIS1_SAME_2D(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS1_2D(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMIN_AXIS1_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half)\n\
\n\
REDUCEMIN_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
"; /* end of reducemin_internal_axis1_vx*/

static const char reducemin_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMIN_PROCESS_AXIS2(read_fun, vert_min_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, min, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_min_fun(min, min, min, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
#define REDUCEMIN_PROCESS_AXIS2_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, min, 16); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_AXIS2_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun) \\\n\
__kernel void reducemin_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS2_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS2(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun) \\\n\
__kernel void reducemin_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMIN_AXIS2_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half)\n\
\n\
\n\
REDUCEMIN_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
\n\
REDUCEMIN_AXIS2(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
\n\
"; /* end of reducemin_internal_axis2_vx*/

static const char reduceprod_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform VXC_512Bits uniGetEndLoData_2x8;\n\
_viv_uniform VXC_512Bits uniGetEndHiData_2x8;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0(read_fun, IN_SCALE, IN_OFFSET) \\\n\
    while(coord.x < inputWidth) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val, val0, 16); \\\n\
        VXC_DP4x4(tmpProdLo, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
        VXC_DP4x4(tmpProdHi, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
        tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProd = tmpProdLo * tmpProdHi; \\\n\
        prodValue = prodValue * tmpProd; \\\n\
        coord.x += 8; \\\n\
    } \\\n\
    vxc_ushort8 tmpProdInt0, tmpProdInt1; \\\n\
    vxc_ushort8 tmpOnesInt = {0, 16256, 0, 16256, 0, 16256, 0, 16256}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, img_val, val0, 16); \\\n\
    VXC_DP4x4(tmpProdLo, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    VXC_DP4x4(tmpProdHi, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
    tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
    _viv_asm(COPY, tmpProdInt0, tmpProdLo, 16); \\\n\
    _viv_asm(COPY, tmpProdInt1, tmpProdHi, 16); \\\n\
    VXC_DP2x8(tmpProdInt0, tmpProdInt0, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndLoData_2x8); \\\n\
    VXC_DP2x8(tmpProdInt1, tmpProdInt1, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndHiData_2x8); \\\n\
    _viv_asm(COPY, tmpProdLo, tmpProdInt0, 16); \\\n\
    _viv_asm(COPY, tmpProdHi, tmpProdInt1, 16); \\\n\
    tmpProd = tmpProdLo * tmpProdHi; \\\n\
    prodValue = prodValue * tmpProd; \\\n\
    tmpProd.xy = prodValue.xy * prodValue.zw; \\\n\
    prodValue.x = tmpProd.x * tmpProd.y;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, write_fun) \\\n\
    dst_type vec1; \\\n\
    save_type dst; \\\n\
    prodValue = prodValue * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, vec1, prodValue); \\\n\
    _viv_asm(COPY, dst, vec1, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEPROD_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
              save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd; \\\n\
    src_type img_val; \\\n\
    copy_type val0; \\\n\
    REDUCEPROD_PROCESS_AXIS0(VXC_ReadImage2DArray, IN_SCALE, IN_OFFSET); \\\n\
    REDUCEPROD_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEPROD_AXIS0(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
\n\
REDUCEPROD_AXIS0(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
#define REDUCEPROD_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(0, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd; \\\n\
    src_type img_val; \\\n\
    copy_type val0; \\\n\
    REDUCEPROD_PROCESS_AXIS0(VXC_ReadImage, IN_SCALE, IN_OFFSET); \\\n\
    REDUCEPROD_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEPROD_AXIS0_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4, \\\n\
vxc_short8, CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
REDUCEPROD_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0_BF16(read_fun) \\\n\
    while(coord.x < inputWidth) \\\n\
    { \\\n\
        read_fun(img_val, input,  coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, tmpProdLo, val0, 16); \\\n\
        VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, tmpProdHi, val0, 16); \\\n\
        tmpProd = tmpProdLo * tmpProdHi; \\\n\
        prodValue = prodValue * tmpProd; \\\n\
        coord.x += 8; \\\n\
    } \\\n\
    vxc_ushort8 tmpProdInt0, tmpProdInt1; \\\n\
    vxc_ushort8 tmpOnesInt = {0, 16256, 0, 16256, 0, 16256, 0, 16256}; \\\n\
    read_fun(img_val, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, tmpProdLo, val0, 16); \\\n\
    VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, tmpProdHi, val0, 16); \\\n\
    _viv_asm(COPY, tmpProdInt0, tmpProdLo, 16); \\\n\
    _viv_asm(COPY, tmpProdInt1, tmpProdHi, 16); \\\n\
    VXC_DP2x8(tmpProdInt0, tmpProdInt0, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndLoData_2x8); \\\n\
    VXC_DP2x8(tmpProdInt1, tmpProdInt1, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndHiData_2x8); \\\n\
    _viv_asm(COPY, tmpProdLo, tmpProdInt0, 16); \\\n\
    _viv_asm(COPY, tmpProdHi, tmpProdInt1, 16); \\\n\
    tmpProd = tmpProdLo * tmpProdHi; \\\n\
    prodValue = prodValue * tmpProd; \\\n\
    tmpProd.xy = prodValue.xy * prodValue.zw; \\\n\
    prodValue.x = tmpProd.x * tmpProd.y;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0_BF16_SAVE(write_fun) \\\n\
    vxc_ushort8 dst; \\\n\
    _viv_asm(COPY, dst, prodValue, 16); \\\n\
    dst.s0 = dst.s1; \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void reduceprod_axis0_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd;\n\
    vxc_short8 img_val;\n\
    vxc_short8 val0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16_SAVE(VXC_WriteImage);\n\
}\n\
\n\
__kernel void reduceprod_axis0_BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(0));\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd;\n\
    vxc_short8 img_val;\n\
    vxc_short8 val0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16(VXC_ReadImage);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16_SAVE(VXC_WriteImage);\n\
}\n\
\n\
"; /* end of reduceprod_internal_axis0_vx*/

static const char reduceprod_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1(read_fun, IN_SCALE, IN_OFFSET) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(tmpProdLo, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
        VXC_DP4x4(tmpProdHi, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
        tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    prodValueLo = prodValueLo * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, prodValueLo); \\\n\
    prodValueHi = prodValueHi * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, prodValueHi); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define REDUCEPROD_AXIS1(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\\\n\
    vxc_float4 tmpProdLo, tmpProdHi; \\\n\
    src_type vec0; \\\n\
    copy_type in0; \\\n\
    REDUCEPROD_PROCESS_AXIS1(VXC_ReadImage2DArray, IN_SCALE, IN_OFFSET) \\\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEPROD_AXIS1(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4, \\\n\
vxc_short8, CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
REDUCEPROD_AXIS1(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
\n\
#define REDUCEPROD_AXIS1_2D(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\\\n\
    vxc_float4 tmpProdLo, tmpProdHi; \\\n\
    src_type vec0; \\\n\
    copy_type in0; \\\n\
    REDUCEPROD_PROCESS_AXIS1(VXC_ReadImage, IN_SCALE, IN_OFFSET) \\\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEPROD_AXIS1_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
REDUCEPROD_AXIS1_2D(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1_BF16(read_fun) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, tmpProdLo, vec0, 16); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, tmpProdHi, vec0, 16); \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1_SAVE_BF16(write_fun) \\\n\
    vxc_ushort8 dst0, dst1; \\\n\
    vxc_ushort8 vect; \\\n\
    _viv_asm(COPY, dst0, prodValueLo, 16); \\\n\
    _viv_asm(COPY, dst1, prodValueHi, 16); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void reduceprod_axis1_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi;\n\
    vxc_short8 vec0;\n\
    vxc_short8 in0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE_BF16(VXC_WriteImage);\n\
}\n\
\n\
__kernel void reduceprod_axis1_BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi;\n\
    vxc_short8 vec0;\n\
    vxc_short8 in0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE_BF16(VXC_WriteImage);\n\
}\n\
"; /* end of reduceprod_internal_axis1_vx*/

static const char reduceprod_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2(read_fun, IN_SCALE, IN_OFFSET) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(tmpProdLo, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
        VXC_DP4x4(tmpProdHi, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
        tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    prodValueLo = prodValueLo * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, prodValueLo); \\\n\
    prodValueHi = prodValueHi * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, prodValueHi); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define REDUCEPROD_AXIS2(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\\\n\
    vxc_float4 tmpProdLo, tmpProdHi; \\\n\
    src_type vec0; \\\n\
    copy_type in0; \\\n\
    REDUCEPROD_PROCESS_AXIS2(VXC_ReadImage2DArray, IN_SCALE, IN_OFFSET) \\\n\
    REDUCEPROD_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET,  VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEPROD_AXIS2(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
\n\
REDUCEPROD_AXIS2(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2_BF16(read_fun) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, tmpProdLo, vec0, 16); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, tmpProdHi, vec0, 16); \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2_SAVE_BF16(write_fun) \\\n\
    vxc_ushort8 dst0, dst1; \\\n\
    vxc_ushort8 vect; \\\n\
    _viv_asm(COPY, dst0, prodValueLo, 16); \\\n\
    _viv_asm(COPY, dst1, prodValueHi, 16); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void reduceprod_axis2_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi;\n\
    vxc_short8 vec0;\n\
    vxc_short8 in0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
    REDUCEPROD_PROCESS_AXIS2_SAVE_BF16(VXC_WriteImage);\n\
}\n\
"; /* end of reduceprod_internal_axis2_vx*/

static const char relational_ops_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float input0Scale;\n\
_viv_uniform float input0Tail;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform float input1Tail;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
#define COMPARISONS_2D(func_name, src0_type_name, src1_type_name, \\\n\
        src0_type, src0_copy_type, src1_type, src1_copy_type, cmp_op) \\\n\
__kernel void func_name##_##src0_type_name##src1_type_name##toBOOL8_2D( \\\n\
    __read_only  image2d_t  input0, \\\n\
    __read_only  image2d_t  input1, \\\n\
    __write_only image2d_t  output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type      src0; \\\n\
    src0_copy_type srcA; \\\n\
    src1_type      src1; \\\n\
    src1_copy_type srcB; \\\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcA, src0, 16); \\\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
 \\\n\
    float4 vecA0, vecA1; \\\n\
    float4 vecB0, vecB1; \\\n\
    VXC_DP4x4(vecA0, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecA1, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA0 = vecA0 * input0Scale + input0Tail; \\\n\
    vecA1 = vecA1 * input0Scale + input0Tail; \\\n\
    VXC_DP4x4(vecB0, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB1, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecB0 = vecB0 * input1Scale + input1Tail; \\\n\
    vecB1 = vecB1 * input1Scale + input1Tail; \\\n\
    int4 dst0, dst1; \\\n\
    dst0 = (vecA0)cmp_op(vecB0); \\\n\
    dst1 = (vecA1)cmp_op(vecB1); \\\n\
 \\\n\
    vxc_char16 dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst &= 1; \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//LESS\n\
COMPARISONS_2D(less, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_2D(less, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_2D(less, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_2D(less, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <)\n\
COMPARISONS_2D(less, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_2D(less, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_2D(less, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_2D(less, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <)\n\
COMPARISONS_2D(less, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <)\n\
COMPARISONS_2D(less, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <)\n\
//GREAT\n\
COMPARISONS_2D(great, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_2D(great, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_2D(great, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_2D(great, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >)\n\
COMPARISONS_2D(great, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_2D(great, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_2D(great, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_2D(great, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >)\n\
COMPARISONS_2D(great, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >)\n\
COMPARISONS_2D(great, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >)\n\
//LESS_EQUAL\n\
COMPARISONS_2D(less_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_2D(less_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_2D(less_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_2D(less_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <=)\n\
COMPARISONS_2D(less_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_2D(less_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_2D(less_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_2D(less_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_2D(less_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <=)\n\
COMPARISONS_2D(less_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <=)\n\
//GREAT_EQUAL\n\
COMPARISONS_2D(great_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_2D(great_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_2D(great_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_2D(great_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >=)\n\
COMPARISONS_2D(great_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_2D(great_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_2D(great_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_2D(great_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_2D(great_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >=)\n\
COMPARISONS_2D(great_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >=)\n\
//EQUAL\n\
COMPARISONS_2D(equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_2D(equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_2D(equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_2D(equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, ==)\n\
COMPARISONS_2D(equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_2D(equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_2D(equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_2D(equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_2D(equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, ==)\n\
COMPARISONS_2D(equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  ==)\n\
//NOT_EQUAL\n\
COMPARISONS_2D(not_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_2D(not_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_2D(not_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_2D(not_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, !=)\n\
COMPARISONS_2D(not_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_2D(not_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_2D(not_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_2D(not_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_2D(not_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, !=)\n\
COMPARISONS_2D(not_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  !=)\n\
\n\
#define COMPARISONS_BF_2D(func_name, src0_type_name, src1_type_name, cmp_op) \\\n\
__kernel void func_name##_##src0_type_name##src1_type_name##toBOOL8_2D( \\\n\
    __read_only  image2d_t  input0, \\\n\
    __read_only  image2d_t  input1, \\\n\
    __write_only image2d_t  output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_ushort8 src0, src1, srcA, srcB; \\\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_ReadImage(srcA, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(srcB, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA0, vecA1; \\\n\
    float4 vecB0, vecB1; \\\n\
    VXC_DP2x8(src0, srcA, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA0, src0, 16); \\\n\
    VXC_DP2x8(src1, srcA, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecA1, src1, 16); \\\n\
    VXC_DP2x8(src0, srcB, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecB0, src0, 16); \\\n\
    VXC_DP2x8(src1, srcB, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB1, src1, 16); \\\n\
    int4 dst0, dst1; \\\n\
    dst0 = (vecA0)cmp_op(vecB0); \\\n\
    dst1 = (vecA1)cmp_op(vecB1); \\\n\
 \\\n\
    vxc_char16 dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst &= 1; \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
COMPARISONS_BF_2D(less,        BF16, BF16,  <)\n\
COMPARISONS_BF_2D(great,       BF16, BF16,  >)\n\
COMPARISONS_BF_2D(less_equal,  BF16, BF16,  <=)\n\
COMPARISONS_BF_2D(great_equal, BF16, BF16,  >=)\n\
COMPARISONS_BF_2D(equal,       BF16, BF16,  ==)\n\
COMPARISONS_BF_2D(not_equal,   BF16, BF16,  !=)\n\
"; /* end of relational_ops_2d_vx*/

static const char relational_ops_3d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float input0Scale;\n\
_viv_uniform float input0Tail;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform float input1Tail;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
#define COMPARISONS_3D(func_name, src0_type_name, src1_type_name, \\\n\
        src0_type, src0_copy_type, src1_type, src1_copy_type, cmp_op) \\\n\
__kernel void func_name##_##src0_type_name##src1_type_name##toBOOL8( \\\n\
    __read_only  image2d_array_t  input0, \\\n\
    __read_only  image2d_array_t  input1, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src0_type      src0; \\\n\
    src0_copy_type srcA; \\\n\
    src1_type      src1; \\\n\
    src1_copy_type srcB; \\\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcA, src0, 16); \\\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
 \\\n\
    float4 vecA0, vecA1, vecA; \\\n\
    float4 vecB0, vecB1, vecB; \\\n\
    VXC_DP4x4(vecA0, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecA1, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA0 = vecA0 * input0Scale + input0Tail; \\\n\
    vecA1 = vecA1 * input0Scale + input0Tail; \\\n\
    VXC_DP4x4(vecB0, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB1, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecB0 = vecB0 * input1Scale + input1Tail; \\\n\
    vecB1 = vecB1 * input1Scale + input1Tail; \\\n\
    int4 dst0, dst1; \\\n\
    dst0 = (vecA0)cmp_op(vecB0); \\\n\
    dst1 = (vecA1)cmp_op(vecB1); \\\n\
 \\\n\
    vxc_char16 dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst &= 1; \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//LESS\n\
COMPARISONS_3D(less, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_3D(less, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_3D(less, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_3D(less, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <)\n\
COMPARISONS_3D(less, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_3D(less, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_3D(less, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_3D(less, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <)\n\
COMPARISONS_3D(less, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <)\n\
COMPARISONS_3D(less, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <)\n\
//GREAT\n\
COMPARISONS_3D(great, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_3D(great, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_3D(great, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_3D(great, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >)\n\
COMPARISONS_3D(great, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_3D(great, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_3D(great, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_3D(great, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >)\n\
COMPARISONS_3D(great, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >)\n\
COMPARISONS_3D(great, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >)\n\
//LESS_EQUAL\n\
COMPARISONS_3D(less_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_3D(less_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_3D(less_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_3D(less_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <=)\n\
COMPARISONS_3D(less_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_3D(less_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_3D(less_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_3D(less_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_3D(less_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <=)\n\
COMPARISONS_3D(less_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <=)\n\
//GREAT_EQUAL\n\
COMPARISONS_3D(great_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_3D(great_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_3D(great_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_3D(great_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >=)\n\
COMPARISONS_3D(great_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_3D(great_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_3D(great_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_3D(great_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_3D(great_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >=)\n\
COMPARISONS_3D(great_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >=)\n\
//EQUAL\n\
COMPARISONS_3D(equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_3D(equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_3D(equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_3D(equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, ==)\n\
COMPARISONS_3D(equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_3D(equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_3D(equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_3D(equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_3D(equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, ==)\n\
COMPARISONS_3D(equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  ==)\n\
//NOT_EQUAL\n\
COMPARISONS_3D(not_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_3D(not_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_3D(not_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_3D(not_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, !=)\n\
COMPARISONS_3D(not_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_3D(not_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_3D(not_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_3D(not_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_3D(not_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, !=)\n\
COMPARISONS_3D(not_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  !=)\n\
\n\
#define COMPARISONS_BF_3D(func_name, src0_type_name, src1_type_name, cmp_op) \\\n\
__kernel void func_name##_##src0_type_name##src1_type_name##toBOOL8_3D( \\\n\
    __read_only  image2d_array_t  input0, \\\n\
    __read_only  image2d_array_t  input1, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    vxc_ushort8 src0, src1, srcA, srcB; \\\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_ReadImage2DArray(srcA, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(srcB, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA0, vecA1; \\\n\
    float4 vecB0, vecB1; \\\n\
    VXC_DP2x8(src0, srcA, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA0, src0, 16); \\\n\
    VXC_DP2x8(src1, srcA, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecA1, src1, 16); \\\n\
    VXC_DP2x8(src0, srcB, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecB0, src0, 16); \\\n\
    VXC_DP2x8(src1, srcB, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB1, src1, 16); \\\n\
    int4 dst0, dst1; \\\n\
    dst0 = (vecA0)cmp_op(vecB0); \\\n\
    dst1 = (vecA1)cmp_op(vecB1); \\\n\
 \\\n\
    vxc_char16 dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst &= 1; \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
COMPARISONS_BF_3D(less,        BF16, BF16,  <)\n\
COMPARISONS_BF_3D(great,       BF16, BF16,  >)\n\
COMPARISONS_BF_3D(less_equal,  BF16, BF16,  <=)\n\
COMPARISONS_BF_3D(great_equal, BF16, BF16,  >=)\n\
COMPARISONS_BF_3D(equal,       BF16, BF16,  ==)\n\
COMPARISONS_BF_3D(not_equal,   BF16, BF16,  !=)\n\
"; /* end of relational_ops_3d_vx*/

static const char relu_keras_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvFP16toFP32_Lo_4x4;\n\
_viv_uniform VXC_512Bits uniConvFP16toFP32_Hi_4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniPackedBF16_2x8;\n\
_viv_uniform VXC_512Bits uniConvIntegertoFP32_Lo_4x4;\n\
_viv_uniform VXC_512Bits uniConvIntegertoFP32_Hi_4x4;\n\
_viv_uniform float offset;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float outputZP;\n\
\n\
float4 I8toF32_Lo(vxc_char8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Lo_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 I8toF32_Hi(vxc_char8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Hi_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 U8toF32_Lo(vxc_uchar8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Lo_4x4);\n\
    dst = dst * input_scale + inputTail;\n\
    return dst;\n\
}\n\
\n\
float4 U8toF32_Hi(vxc_uchar8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Hi_4x4);\n\
    dst = dst * input_scale + inputTail;\n\
    return dst;\n\
}\n\
\n\
float4 I16toF32_Lo(vxc_short8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Lo_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 I16toF32_Hi(vxc_short8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Hi_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 F16toF32_Lo(vxc_ushort8 src)\n\
{\n\
    vxc_half8 srcHalf;\n\
    float4 dst;\n\
\n\
    _viv_asm(COPY, srcHalf, src, 16);\n\
    VXC_DP4x4(dst, srcHalf, srcHalf, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvFP16toFP32_Lo_4x4);\n\
    return dst;\n\
}\n\
\n\
float4 F16toF32_Hi(vxc_ushort8 src)\n\
{\n\
    vxc_half8 srcHalf;\n\
    float4 dst;\n\
\n\
    _viv_asm(COPY, srcHalf, src, 16);\n\
    VXC_DP4x4(dst, srcHalf, srcHalf, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvFP16toFP32_Hi_4x4);\n\
    return dst;\n\
}\n\
\n\
float4 BF16toF32_Lo(vxc_ushort8 src)\n\
{\n\
    vxc_ushort8 srcA;\n\
    float4 dst;\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(srcA, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, dst, srcA, 16);\n\
\n\
    return dst;\n\
}\n\
\n\
float4 BF16toF32_Hi(vxc_ushort8 src)\n\
{\n\
    vxc_ushort8 srcA;\n\
    float4 dst;\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(srcA, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, dst, srcA, 16);\n\
\n\
    return dst;\n\
}\n\
\n\
vxc_ushort8 F32toF16(float4 src0, float4 src1)\n\
{\n\
    half4 srcHalf0, srcHalf1;\n\
    vxc_half8 dst0;\n\
    vxc_ushort8 dst;\n\
\n\
    _viv_asm(CONV, srcHalf0, src0);\n\
    _viv_asm(CONV, srcHalf1, src1);\n\
\n\
    VXC_DP2x8(dst0, srcHalf0, srcHalf1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
    _viv_asm(COPY, dst, dst0, 16);\n\
    return dst;\n\
}\n\
\n\
vxc_ushort8 F32toBF16(float4 src0, float4 src1)\n\
{\n\
    vxc_ushort8 srcA, srcB;\n\
    vxc_ushort8 dst;\n\
\n\
    _viv_asm(COPY, srcA, src0, 16);\n\
    _viv_asm(COPY, srcB, src1, 16);\n\
    VXC_DP2x8(dst, srcA, srcB, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackedBF16_2x8);\n\
    return dst;\n\
}\n\
\n\
vxc_char8 F32toI8(float4 src0, float4 src1)\n\
{\n\
    int4 srcInt0, srcInt1;\n\
    vxc_char8 dst;\n\
\n\
    src0 *= output_scale;\n\
    src1 *= output_scale;\n\
    _viv_asm(CONV_RTE, srcInt0, src0);\n\
    _viv_asm(CONV_RTE, srcInt1, src1);\n\
\n\
    VXC_DP2x8(dst, srcInt0, srcInt1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInteger_2x8);\n\
    return dst;\n\
}\n\
\n\
vxc_short8 F32toI16(float4 src0, float4 src1)\n\
{\n\
    int4 srcInt0, srcInt1;\n\
    vxc_short8 dst;\n\
\n\
    src0 *= output_scale;\n\
    src1 *= output_scale;\n\
    _viv_asm(CONV_RTE, srcInt0, src0);\n\
    _viv_asm(CONV_RTE, srcInt1, src1);\n\
\n\
    VXC_DP2x8(dst, srcInt0, srcInt1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInteger_2x8);\n\
    return dst;\n\
}\n\
\n\
vxc_uchar8 F32toU8(float4 src0, float4 src1)\n\
{\n\
    int4 srcInt0, srcInt1;\n\
    vxc_uchar8 dst;\n\
\n\
    src0 = src0 * output_scale + outputZP;\n\
    src1 = src1 * output_scale + outputZP;\n\
    _viv_asm(CONV_RTE, srcInt0, src0);\n\
    _viv_asm(CONV_RTE, srcInt1, src1);\n\
\n\
    VXC_DP2x8(dst, srcInt0, srcInt1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInteger_2x8);\n\
    return dst;\n\
}\n\
\n\
\n\
#define TENSOR_KERAS_RELU(src_type_name, dst_type_name, tensor_dims, image_type, \\\n\
    convert2FP32_Func, convert2DstType_Func, src_type, dst_type) \\\n\
__kernel void relu_keras_##src_type_name##to##dst_type_name##tensor_dims( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
                    float     alpha, \\\n\
                    float     max_value, \\\n\
                    float     threshold \\\n\
                    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    VXC_Read##image_type(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 dataA = convert2FP32_Func##_Lo(src); \\\n\
    float4 dataB = convert2FP32_Func##_Hi(src); \\\n\
    float4 dataA0 = dataA < threshold ? threshold : dataA; \\\n\
    dataA0 = dataA0 > max_value ? max_value : dataA0; \\\n\
    float4 dataB0 = dataB < threshold ? threshold : dataB; \\\n\
    dataB0 = dataB0 > max_value ? max_value : dataB0; \\\n\
    float4 dataA1 = dataA * alpha - offset; \\\n\
    float4 dataB1 = dataB * alpha - offset; \\\n\
    float4 dst0 = dataA  < threshold ? dataA1 : dataA0; \\\n\
    float4 dst1 = dataB  < threshold ? dataB1 : dataB0; \\\n\
    dst_type result = convert2DstType_Func(dst0, dst1); \\\n\
    VXC_Write##image_type(output, coord, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
TENSOR_KERAS_RELU(F16,  F16,  _3D, Image2DArray, F16toF32,  F32toF16,  vxc_ushort8, vxc_ushort8)\n\
TENSOR_KERAS_RELU(F16,  I16,  _3D, Image2DArray, F16toF32,  F32toI16,  vxc_ushort8, vxc_short8)\n\
TENSOR_KERAS_RELU(F16,  I8,   _3D, Image2DArray, F16toF32,  F32toI8,   vxc_ushort8, vxc_char8)\n\
TENSOR_KERAS_RELU(F16,  U8,   _3D, Image2DArray, F16toF32,  F32toU8,   vxc_ushort8, vxc_uchar8)\n\
TENSOR_KERAS_RELU(BF16, BF16, _3D, Image2DArray, BF16toF32, F32toBF16, vxc_ushort8, vxc_ushort8)\n\
\n\
TENSOR_KERAS_RELU(I16,  I16,  _3D, Image2DArray, I16toF32,  F32toI16,  vxc_short8,  vxc_short8)\n\
TENSOR_KERAS_RELU(I16,  F16,  _3D, Image2DArray, I16toF32,  F32toF16,  vxc_short8,  vxc_ushort8)\n\
TENSOR_KERAS_RELU(I8,   I8,   _3D, Image2DArray, I8toF32,   F32toI8,   vxc_char8,   vxc_char8)\n\
TENSOR_KERAS_RELU(I8,   F16,  _3D, Image2DArray, I8toF32,   F32toF16,  vxc_char8,   vxc_ushort8)\n\
TENSOR_KERAS_RELU(U8,   U8,   _3D, Image2DArray, U8toF32,   F32toU8,   vxc_uchar8,  vxc_uchar8)\n\
TENSOR_KERAS_RELU(U8,   F16,  _3D, Image2DArray, U8toF32,   F32toF16,  vxc_uchar8,  vxc_ushort8)\n\
\n\
TENSOR_KERAS_RELU(F16,  F16,  _2D, Image,        F16toF32,  F32toF16,  vxc_ushort8, vxc_ushort8)\n\
TENSOR_KERAS_RELU(F16,  I16,  _2D, Image,        F16toF32,  F32toI16,  vxc_ushort8, vxc_short8)\n\
TENSOR_KERAS_RELU(F16,  I8,   _2D, Image,        F16toF32,  F32toI8,   vxc_ushort8, vxc_char8)\n\
TENSOR_KERAS_RELU(F16,  U8,   _2D, Image,        F16toF32,  F32toU8,   vxc_ushort8, vxc_uchar8)\n\
TENSOR_KERAS_RELU(BF16, BF16, _2D, Image,        BF16toF32, F32toBF16, vxc_ushort8, vxc_ushort8)\n\
TENSOR_KERAS_RELU(I16,  I16,  _2D, Image,        I16toF32,  F32toI16,  vxc_short8,  vxc_short8)\n\
TENSOR_KERAS_RELU(I16,  F16,  _2D, Image,        I16toF32,  F32toF16,  vxc_short8,  vxc_ushort8)\n\
TENSOR_KERAS_RELU(I8,   I8,   _2D, Image,        I8toF32,   F32toI8,   vxc_char8,   vxc_char8)\n\
TENSOR_KERAS_RELU(I8,   F16,  _2D, Image,        I8toF32,   F32toF16,  vxc_char8,   vxc_ushort8)\n\
TENSOR_KERAS_RELU(U8,   U8,   _2D, Image,        U8toF32,   F32toU8,   vxc_uchar8,  vxc_uchar8)\n\
TENSOR_KERAS_RELU(U8,   F16,  _2D, Image,        U8toF32,   F32toF16,  vxc_uchar8,  vxc_ushort8)\n\
"; /* end of relu_keras_vx*/

static const char repeat_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniIntegralHorAcc_4x4;\n\
_viv_uniform VXC_512Bits uniExtract1to8Short_2x8;\n\
_viv_uniform int width;\n\
\n\
// workgroup size is 32\n\
__kernel void preprocess_start_idx(image2d_t input, image2d_t output)\n\
{\n\
    int lidx = get_local_id(0);\n\
    __local int lcl_sum[32];\n\
    __local int last_round[1];\n\
    Image img = create_image_from_image2d(input, 4);\n\
    Image dst = create_image_from_image2d(output, 4);\n\
    __global int* index_ptr = (__global int*)img.ptr + get_global_id(0);\n\
    __global int* output_org = (__global int*)dst.ptr;\n\
    __global int* output_ptr = output_org + get_global_id(0) + 1;\n\
\n\
    if (lidx == 0)\n\
    {\n\
        last_round[0] = 0;\n\
        output_org[0] = 0;\n\
    }\n\
    int4 accSum0, accSum1, accSum2, accSum3;\n\
\n\
    for(int i = 0; i < width; i += 512)\n\
    {\n\
        int4 data0 = vload4(0, index_ptr + i);\n\
        int4 data1 = vload4(1, index_ptr + i);\n\
        int4 data2 = vload4(2, index_ptr + i);\n\
        int4 data3 = vload4(3, index_ptr + i);\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
        int prevSum = last_round[0];\n\
\n\
        VXC_DP4x4(accSum0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        VXC_DP4x4(accSum1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        VXC_DP4x4(accSum2, data2, data2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        VXC_DP4x4(accSum3, data3, data3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        accSum1 += accSum0.w;\n\
        accSum2 += accSum1.w;\n\
        accSum3 += accSum2.w;\n\
\n\
        lcl_sum[lidx] = accSum3.w;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        for(int j = 0; j < lidx; j++)\n\
        {\n\
            prevSum += lcl_sum[j];\n\
        }\n\
        accSum0 += prevSum;\n\
        accSum1 += prevSum;\n\
        accSum2 += prevSum;\n\
        accSum3 += prevSum;\n\
        if(lidx == 31)\n\
        {\n\
            last_round[0] = accSum3.w;\n\
        }\n\
        vstore4(accSum0, 0, output_ptr + i);\n\
        vstore4(accSum1, 1, output_ptr + i);\n\
        vstore4(accSum2, 2, output_ptr + i);\n\
        vstore4(accSum3, 3, output_ptr + i);\n\
    }\n\
}\n\
\n\
__kernel void repeat_I16_axis0(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    vxc_short8 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(1)];\n\
    int start = index_ptr[get_global_id(1)];\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int end = len + start;\n\
    coord.w = get_global_id(2);\n\
\n\
    for(coord.y = start; coord.y < end; coord.y++)\n\
    {\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void repeat_I16_axis2(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(2)];\n\
    int start = index_ptr[get_global_id(2)];\n\
    int end = len + start;\n\
\n\
    for(coord.z = start; coord.z < end; coord.z++)\n\
    {\n\
        VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define REPEAT_1D(src0_type_name, data_type) \\\n\
__kernel void repeat_##src0_type_name##_1D( \\\n\
    image2d_t  input0, image2d_t  input1, image2d_t  input2, \\\n\
    image2d_t  output, int axis) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    data_type src0; \\\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
             VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    Image img2 = create_image_from_image2d(input2, 4); \\\n\
    __global int* len_ptr = (__global int*)img1.ptr; \\\n\
    __global int* index_ptr = (__global int*)img2.ptr; \\\n\
    int len = len_ptr[get_global_id(0)]; \\\n\
    int start = index_ptr[get_global_id(0)]; \\\n\
 \\\n\
    int iter = len >> 3; \\\n\
    int res = len & 7; \\\n\
    int end = start + iter * 8; \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8); \\\n\
    for(coord.x = start; coord.x < end; coord.x+=8) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
 \\\n\
    if(res == 7) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 6) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 5) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 4) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 3) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 2) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 1) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
REPEAT_1D(U8,  vxc_uchar16)\n\
REPEAT_1D(I16, vxc_short8)\n\
\n\
__kernel void repeat_U8_axis0(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_uchar16 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(1)];\n\
    int start = index_ptr[get_global_id(1)];\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int end = len + start;\n\
    coord.w = get_global_id(2);\n\
\n\
    for(coord.y = start; coord.y < end; coord.y++)\n\
    {\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, \\\n\
                VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void repeat_U8_axis2(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_uchar16 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(2)];\n\
    int start = index_ptr[get_global_id(2)];\n\
    int end = len + start;\n\
\n\
    for(coord.z = start; coord.z < end; coord.z++)\n\
    {\n\
        VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of repeat_vx*/

static const char repeat_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtract1to8Short_2x8;\n\
\n\
#define REPEAT_RES(end_pos) \\\n\
coord.y = gidy; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src0, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src1, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src3, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src4, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src5, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src6, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src7, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void repeat_I16_axis1(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), gidy, get_global_id(2), get_global_id(2));\n\
    vxc_short8 src0, src1, src2, src3, src4, src5, src6, src7;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
\n\
    VXC_OP4(img_load_3d, src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input0, coord, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input0, coord, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input0, coord, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src4, input0, coord, VXC_5BITOFFSET_XY(0, 4),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src5, input0, coord, VXC_5BITOFFSET_XY(0, 5),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src6, input0, coord, VXC_5BITOFFSET_XY(0, 6),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src7, input0, coord, VXC_5BITOFFSET_XY(0, 7),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(0)];\n\
    int start = index_ptr[get_global_id(0)];\n\
\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int iter = len >> 3;\n\
    int res = len & 7;\n\
    coord.x = start;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src2, src2, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src3, src3, src3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src4, src4, src4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src5, src5, src5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src6, src6, src6, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src7, src7, src7, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        coord.y = gidy;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src1, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src3, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src4, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src5, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src6, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src7, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
    }\n\
\n\
    if(res == 7)\n\
    {\n\
        REPEAT_RES(6)\n\
    }\n\
    else if(res == 6)\n\
    {\n\
        REPEAT_RES(5)\n\
    }\n\
    else if(res == 5)\n\
    {\n\
        REPEAT_RES(4)\n\
    }\n\
    else if(res == 4)\n\
    {\n\
        REPEAT_RES(3)\n\
    }\n\
    else if(res == 3)\n\
    {\n\
        REPEAT_RES(2)\n\
    }\n\
    else if(res == 2)\n\
    {\n\
        REPEAT_RES(1)\n\
    }\n\
    else if(res == 1)\n\
    {\n\
        REPEAT_RES(0)\n\
    }\n\
}\n\
\n\
__kernel void repeat_U8_axis1(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), gidy, get_global_id(2), get_global_id(2));\n\
    vxc_uchar16 src0, src1, src2, src3, src4, src5, src6, src7;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
\n\
    VXC_OP4(img_load_3d, src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input0, coord, VXC_5BITOFFSET_XY(0, 1),\n\
             VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input0, coord, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input0, coord, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src4, input0, coord, VXC_5BITOFFSET_XY(0, 4),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src5, input0, coord, VXC_5BITOFFSET_XY(0, 5),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src6, input0, coord, VXC_5BITOFFSET_XY(0, 6),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src7, input0, coord, VXC_5BITOFFSET_XY(0, 7),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(0)];\n\
    int start = index_ptr[get_global_id(0)];\n\
\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int iter = len >> 3;\n\
    int res = len & 7;\n\
    coord.x = start;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src2, src2, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src3, src3, src3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src4, src4, src4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src5, src5, src5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src6, src6, src6, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src7, src7, src7, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        coord.y = gidy;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src1, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src3, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src4, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src5, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src6, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src7, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
    }\n\
\n\
    if(res == 7)\n\
    {\n\
        REPEAT_RES(6)\n\
    }\n\
    else if(res == 6)\n\
    {\n\
        REPEAT_RES(5)\n\
    }\n\
    else if(res == 5)\n\
    {\n\
        REPEAT_RES(4)\n\
    }\n\
    else if(res == 4)\n\
    {\n\
        REPEAT_RES(3)\n\
    }\n\
    else if(res == 3)\n\
    {\n\
        REPEAT_RES(2)\n\
    }\n\
    else if(res == 2)\n\
    {\n\
        REPEAT_RES(1)\n\
    }\n\
    else if(res == 1)\n\
    {\n\
        REPEAT_RES(0)\n\
    }\n\
}\n\
\n\
"; /* end of repeat_axis1_vx*/

static const char resize_1d_bilinear_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_odd_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_even_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_1d_bilinear_BF16toBF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x      = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x         = (convert_float4(coord_x) + half_pixel_value)  * scale_x - half_pixel_value;\n\
    float4 left_x_f     = floor(in_x);\n\
    float4 x_lerp       = in_x - left_x_f;\n\
    int4   left_x_idx   = convert_int4(left_x_f);\n\
\n\
    vxc_short8 top;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.x;\n\
\n\
        vxc_ushort8 src;\n\
        float4 left4;\n\
        float4 right4;\n\
        float4 dst4;\n\
\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8);\n\
        _viv_asm(COPY, right4, src, 16);\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8);\n\
        _viv_asm(COPY, left4,  src, 16);\n\
        right4    -= left4;\n\
        dst4       = right4 * x_lerp + left4;\n\
        vxc_ushort8 tmp, dst;\n\
        _viv_asm(COPY, tmp, dst4, 16);\n\
        dst.s0123 = tmp.s1357;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_BF16toBF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.y ++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_ushort8 dst_tmp;\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, left4, dst_tmp, 16);\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, right4, dst_tmp, 16);\n\
        right4     -= left4;\n\
        float4 dst4 = right4 * x_lerp + left4;\n\
\n\
        vxc_ushort8 tmp, dst;\n\
        _viv_asm(COPY, tmp, dst4, 16);\n\
        dst.s0123 = tmp.s1357;\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
"; /* end of resize_1d_bilinear_BF16_vx*/

static const char resize_1d_bilinear_DOWN_NX_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResizeNxDown_2x8;\n\
_viv_uniform int out_height;\n\
\n\
#define RESIZE_1D_NX_DOWN_8BIT_SAME_PROCESS(read_type, data_type) \\\n\
    read_type read_data, save_data; \\\n\
    data_type in0, result; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    while (coord_out.y < out_height) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, read_data, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in0, read_data, 16); \\\n\
        VXC_DP2x8(result, in0, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResizeNxDown_2x8); \\\n\
        _viv_asm(COPY, save_data, result, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, save_data, \\\n\
            VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y++; \\\n\
        coord_out.y++; \\\n\
    } \\\n\
\n\
#define RESIZE_1D_2X_DOWN_8BIT_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_8BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_8BIT_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_DOWN_8BIT_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
\n\
\n\
\n\
#define RESIZE_1D_2X_DOWN_8BIT_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_8BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_8BIT_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_DOWN_8BIT_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
\n\
\n\
#define RESIZE_1D_NX_DOWN_16BIT_SAME_PROCESS(read_type, data_type) \\\n\
    read_type read_data, read_data1, save_data; \\\n\
    data_type in0, in1, result; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    while (coord_out.y < out_height) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, read_data, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in0, read_data, 16); \\\n\
        VXC_OP4(img_load_3d, read_data1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in1, read_data1, 16); \\\n\
        VXC_DP2x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResizeNxDown_2x8); \\\n\
        _viv_asm(COPY, save_data, result, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, save_data, \\\n\
            VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y++; \\\n\
        coord_out.y++; \\\n\
    } \\\n\
\n\
#define RESIZE_1D_2X_DOWN_16BIT_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_16BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_16BIT_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_DOWN_16BIT_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
\n\
#define RESIZE_1D_2X_DOWN_16BIT_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_16BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_16BIT_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_DOWN_16BIT_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
"; /* end of resize_1d_bilinear_DOWN_NX_vx*/

static const char resize_1d_bilinear_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform VXC_512Bits uniConvertFp2FP32_left_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFp2FP32_right_4x4;\n\
\n\
__kernel void resize_1d_bilinear_F16toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 left4;\n\
    float4 right4;\n\
\n\
    vxc_ushort8 src, result;\n\
    vxc_half8 src_half, dst;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.x;\n\
\n\
        _viv_asm(COPY, src_half, src, 16);\n\
\n\
        VXC_DP4x4(left4,  src_half, src_half, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniConvertFp2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src_half, src_half, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniConvertFp2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        half4 tmp;\n\
        _viv_asm(CONV, tmp, dst4);\n\
        VXC_DP2x8(dst, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, result, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
\n\
}\n\
\n\
__kernel void resize_1d_bilinear_F16toU8_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 left4;\n\
    float4 right4;\n\
\n\
    vxc_ushort8 src;\n\
    vxc_uchar8 result;\n\
    vxc_half8 src_half, dst;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.x;\n\
\n\
        _viv_asm(COPY, src_half, src, 16);\n\
\n\
        VXC_DP4x4(left4,  src_half, src_half, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniConvertFp2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src_half, src_half, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniConvertFp2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_F16toF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_half8 top;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
\n\
        coord_in.y ++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        half4 tmp;\n\
        _viv_asm(CONV, tmp, dst4);\n\
        VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, dst0, top, 16);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
"; /* end of resize_1d_bilinear_F16_vx*/

static const char resize_1d_bilinear_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_4x4;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_left_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_right_4x4;\n\
\n\
__kernel void resize_1d_bilinear_I16toI16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
\n\
    vxc_short8 top;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        coord_in.y ++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_part1_4x4);\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4         = dst4 * dfpScale;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_I16toI16_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
    vxc_short8 result;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.x;\n\
\n\
        VXC_DP4x4(left4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4         = dst4 * dfpScale;\n\
\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
"; /* end of resize_1d_bilinear_I16_vx*/

static const char resize_1d_bilinear_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_4x4;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_left_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_right_4x4;\n\
\n\
__kernel void resize_1d_bilinear_I8toI8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_uchar16 src0, dst0;\n\
\n\
    vxc_char16 top;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
\n\
        coord_in.y++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        VXC_DP4x4(left4, top, top, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_4x4);\n\
        VXC_DP4x4(right4, top, top, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_part1_4x4);\n\
\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        dst4         = dst4 * dfpScale;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_I8toI8_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_char16 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
    vxc_char16 result;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.x;\n\
\n\
        VXC_DP4x4(left4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4         = dst4 * dfpScale;\n\
\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
\n\
}\n\
"; /* end of resize_1d_bilinear_I8_vx*/

static const char resize_1d_bilinear_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_right_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_1d_bilinear_U8toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    float4 one_minus_lerp = 1 - x_lerp;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
\n\
    x_lerp = x_lerp * uint8Scale;\n\
    one_minus_lerp = one_minus_lerp * uint8Scale;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.x;\n\
\n\
        VXC_DP4x4(left4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_right_4x4);\n\
        float4 dst4  = right4 * x_lerp + left4 * one_minus_lerp;\n\
        half4 dst;\n\
        _viv_asm(CONV, dst, dst4);\n\
        vxc_short8 dst_short;\n\
        _viv_asm(COPY, dst_short, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst_short,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_U8toU8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_uchar16 src0, src1;\n\
\n\
    vxc_uchar16 top;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(top, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        unsigned char inputZP;\n\
        _viv_asm(COPY, inputZP, input_ZP, 4);\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
        float4 dst4 = right4 * x_lerp + left4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_U8toU8_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    float4 one_minus_lerp = 1 - x_lerp;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
    vxc_uchar16 result;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    x_lerp = x_lerp * uint8Scale;\n\
    one_minus_lerp = one_minus_lerp * uint8Scale;\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.x;\n\
\n\
        VXC_DP4x4(left4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_right_4x4);\n\
        float4 dst4  = right4 * x_lerp + left4 * one_minus_lerp + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
\n\
}\n\
"; /* end of resize_1d_bilinear_U8_vx*/

static const char resize_1d_bilinear_U8_opt_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4x4;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_1d_bilinear_U8toU8_UP_opt\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers,\n\
    __read_only  image2d_array_t   scale\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    int4   right_x_idx = left_x_idx + 1;\n\
\n\
    vxc_uchar16 src0;\n\
\n\
    vxc_uchar16 src_mask;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    vxc_ushort8 lerp_0;\n\
    vxc_half8 lerp;\n\
\n\
    int2 coord = (int2)(coord_out.x * 2, 0);\n\
    VXC_ReadImage(lerp_0, scale, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, lerp, lerp_0, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(src_mask, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.y++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_uchar16 dst;\n\
        VXC_DP4x4(dst, src_mask, lerp,\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniBilinear_4x4);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
"; /* end of resize_1d_bilinear_U8_opt_vx*/

static const char resize_1d_bilinear_UP_NX_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResizeNxUp_2x8;\n\
_viv_uniform int out_height;\n\
\n\
#define RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
    read_type read_data, save_data; \\\n\
    data_type in0, result; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    while (coord_out.y < out_height) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, read_data, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in0, read_data, 16); \\\n\
        VXC_DP2x8(result, in0, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResizeNxUp_2x8); \\\n\
        _viv_asm(COPY, save_data, result, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, save_data, \\\n\
            VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y++; \\\n\
        coord_out.y++; \\\n\
    } \\\n\
\n\
#define RESIZE_1D_2X_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_2X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x = (coord_out.x * 2 - 1) >> 2; \\\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_2X_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_2X_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_2X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x >> 1; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_2X_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_4X_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_4X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x = (coord_out.x * 2 - 3) >> 3; \\\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_4X_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_4X_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_4X_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_4X_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_4X_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_4X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x >> 2; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_4X_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_4X_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_4X_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_4X_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_8X_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_8X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x = (coord_out.x * 2 - 7) >> 4; \\\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_8X_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_8X_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_8X_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_8X_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_8X_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_8X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x >> 3; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_8X_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_8X_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_8X_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_8X_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
"; /* end of resize_1d_bilinear_UP_NX_vx*/

static const char resize_1d_nearest_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniMultiplyAndPostShift_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform float round_value;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4   coord_x    = coord_out.xxxx + (int4)(0, 1, 2, 3); \\\n\
    float4 in_x       = (convert_float4(coord_x) + half_pixel_value) * scale_x + round_value; \\\n\
    int4   in_x_idx   = convert_int4(in_x); \\\n\
\n\
\n\
__kernel void resize_1d_nearest_F16toF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniGetExtractData_2x8;\n\
__kernel void resize_1d_nearest_F16toF16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_2x8;\n\
__kernel void resize_1d_nearest_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_char16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_I8toI8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst0;\n\
    vxc_char16 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_U8toU8\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(src, src, multiplier, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_U8toU8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(dst, dst, multiplier, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_I16toI16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_I16toI16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_short8 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_1d_nearest_vx*/

static const char resize_bilinear_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_odd_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_even_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_BF16toBF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x      = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x         = (convert_float4(coord_x) + half_pixel_value)  * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f     = floor(in_x);\n\
    float4 x_lerp       = in_x - left_x_f;\n\
    int4   left_x_idx   = convert_int4(left_x_f);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float  y_lerp       = in_y - top_y_f;\n\
    int    top_y_idx    = convert_int(top_y_f);\n\
    vxc_short8 top;\n\
    vxc_short8 bottom;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 src;\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
    float4 dst4;\n\
\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8);\n\
    _viv_asm(COPY, right4, src, 16);\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8);\n\
    _viv_asm(COPY, left4,  src, 16);\n\
    right4    -= left4;\n\
    top4       = right4 * x_lerp + left4;\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8);\n\
    _viv_asm(COPY, right4, src, 16);\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8);\n\
    _viv_asm(COPY, left4,  src, 16);\n\
    right4    -= left4;\n\
    bottom4    = right4 * x_lerp + left4;\n\
    bottom4   -= top4;\n\
    dst4       = bottom4 * y_lerp + top4;\n\
    vxc_ushort8 tmp, dst;\n\
    _viv_asm(COPY, tmp, dst4, 16);\n\
    dst.s0123 = tmp.s1357;\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_BF16toBF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_ushort8 src0, src1, src2, src3, dst0, dst1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_ushort8 dst_tmp;\n\
\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, left4, dst_tmp, 16);\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, right4, dst_tmp, 16);\n\
        right4     -= left4;\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, left4, dst_tmp, 16);\n\
        VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, right4, dst_tmp, 16);\n\
        right4     -= left4;\n\
        bottom4     = right4 * x_lerp + left4;\n\
\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
        vxc_ushort8 tmp, dst;\n\
        _viv_asm(COPY, tmp, dst4, 16);\n\
        dst.s0123 = tmp.s1357;\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 dst_tmp;\n\
    VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, left4, dst_tmp, 16);\n\
    VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, right4, dst_tmp, 16);\n\
    right4     -= left4;\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, left4, dst_tmp, 16);\n\
    VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, right4, dst_tmp, 16);\n\
    right4     -= left4;\n\
    bottom4     = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    vxc_ushort8 tmp, dst;\n\
    _viv_asm(COPY, tmp, dst4, 16);\n\
    dst.s0123 = tmp.s1357;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_BF16_vx*/

static const char resize_bilinear_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_F16toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_short8 top_short, bottom_short, dst;\n\
    vxc_half8  top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top,    top_short, 16);\n\
    _viv_asm(COPY, bottom, bottom_short, 16);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4       = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4    = right4 * x_lerp + left4;\n\
    bottom4   -= top4;\n\
    float4 dst4       = bottom4 * y_lerp + top4;\n\
\n\
    half4 tmp;\n\
    _viv_asm(CONV, tmp, dst4);\n\
    VXC_DP2x8(result, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
    _viv_asm(COPY, dst, result, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define RESIZE_BILINEAR_F16TOQINT_DOWN(out_name, dst_type) \\\n\
__kernel void resize_bilinear_F16to##out_name##_DOWN( \\\n\
    __read_only     image2d_array_t input, \\\n\
    __write_only    image2d_array_t output, \\\n\
    int   align_corners, \\\n\
    int   half_pixel_centers \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3); \\\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value; \\\n\
    float4 left_x_f    = floor(in_x); \\\n\
    float4 x_lerp      = in_x - left_x_f; \\\n\
    int4   left_x_idx  = convert_int4(left_x_f); \\\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value; \\\n\
    float  top_y_f     = floor(in_y); \\\n\
    float  y_lerp      = in_y - top_y_f; \\\n\
    int    top_y_idx   = convert_int(top_y_f); \\\n\
 \\\n\
    vxc_short8 top_short, bottom_short; \\\n\
    vxc_half8  top, bottom; \\\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0); \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = left_x_idx.y; \\\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = left_x_idx.z; \\\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.x = left_x_idx.w; \\\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, top,    top_short, 16); \\\n\
    _viv_asm(COPY, bottom, bottom_short, 16); \\\n\
 \\\n\
    float4 left4, right4, top4, bottom4; \\\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    top4        = right4 * x_lerp + left4; \\\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4); \\\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4); \\\n\
    bottom4      = right4 * x_lerp + left4; \\\n\
    bottom4     -= top4; \\\n\
    float4 dst4  = bottom4 * y_lerp + top4; \\\n\
    dst4         = dst4 * uint8Scale + output_ZP; \\\n\
    int4 dst     = convert_int4_rte(dst4); \\\n\
    dst_type dst_uchar; \\\n\
    VXC_DP2x8(dst_uchar, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst_uchar, \\\n\
        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
RESIZE_BILINEAR_F16TOQINT_DOWN(U8, vxc_uchar8)\n\
RESIZE_BILINEAR_F16TOQINT_DOWN(U16, vxc_ushort8)\n\
\n\
__kernel void resize_bilinear_F16toF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
\n\
    vxc_ushort8 src0, src1, src2, src3, dst0, dst1;\n\
    vxc_half8 top;\n\
    vxc_half8 bottom;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
        _viv_asm(COPY, bottom, dst1, 16);\n\
\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
        top4        = right4 * x_lerp + left4;\n\
        VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        half4 tmp;\n\
        _viv_asm(CONV, tmp, dst4);\n\
        VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, dst0, top, 16);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top, dst0, 16);\n\
    _viv_asm(COPY, bottom, dst1, 16);\n\
\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    half4 tmp;\n\
    _viv_asm(CONV, tmp, dst4);\n\
    VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
    _viv_asm(COPY, dst0, top, 16);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_F16_vx*/

static const char resize_bilinear_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_I16toI16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_ushort8 src0, src1, src2, src3, dst0, dst1;\n\
\n\
    vxc_short8 top;\n\
    vxc_short8 bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4, right4, top4, bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
        _viv_asm(COPY, bottom, dst1, 16);\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP4x4(left4, bottom, inputZP, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top, dst0, 16);\n\
    _viv_asm(COPY, bottom, dst1, 16);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
}\n\
\n\
__kernel void resize_bilinear_I16toI16_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_short8 top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
\n\
    float4 left4, right4, top4, bottom4;\n\
\n\
    VXC_DP4x4(left4, top, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                    VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of resize_bilinear_I16_vx*/

static const char resize_bilinear_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_I8toI8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_uchar16 src0, src1, dst0, dst1;\n\
\n\
    vxc_char16 top;\n\
    vxc_char16 bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
        _viv_asm(COPY, bottom, dst1, 16);\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, inputZP, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP4x4(left4, bottom, inputZP, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top, dst0, 16);\n\
    _viv_asm(COPY, bottom, dst1, 16);\n\
    VXC_DP4x4(left4, top, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_I8toI8_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_char16 top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
\n\
    int4 dst     = convert_int4_rte(dst4);\n\
\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_I8_vx*/

static const char resize_bilinear_U16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_U16toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_ushort8 top, bottom;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4, right4, top4, bottom4;\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4 *=  uint8Scale;\n\
\n\
    half4 dst;\n\
    _viv_asm(CONV, dst, dst4);\n\
\n\
    vxc_short8 dst_short;\n\
    _viv_asm(COPY, dst_short, dst, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst_short.s0246,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_U16toU16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_ushort8 src0, src1, src2, src3;\n\
\n\
    vxc_ushort8 top;\n\
    vxc_ushort8 bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4, right4, top4, bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(top, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(bottom, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP4x4(left4, bottom, inputZP, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(top, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(bottom, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_U16toU16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_ushort8 top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4, right4, top4, bottom4;\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_U16_vx*/

static const char resize_bilinear_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_U8toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_uchar16 top, bottom;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4 *=  uint8Scale;\n\
\n\
    half4 dst;\n\
    _viv_asm(CONV, dst, dst4);\n\
\n\
    vxc_short8 dst_short;\n\
    _viv_asm(COPY, dst_short, dst, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst_short.s0246,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_U8toU8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_uchar16 src0, src1;\n\
\n\
    vxc_uchar16 top;\n\
    vxc_uchar16 bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    short inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(top, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(bottom, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP4x4(left4, bottom, inputZP, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(top, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(bottom, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_U8toU8_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_uchar16 top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_U8_vx*/

static const char resize_bilinear_U8_half_pixel_centers_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResize2xUp_0_4x8;\n\
_viv_uniform VXC_512Bits uniResize2xUp_1_4x8;\n\
_viv_uniform int out_height;\n\
\n\
__kernel void resize_bilinear_U8toU8_SAME_2x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 1) >> 2;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, tmp, result;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        coord_out.y++;\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize4xUp_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l01_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l10_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l11_4x8;\n\
__kernel void resize_bilinear_U8toU8_SAME_4x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 3) >> 3;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, tmp, dst0, dst1;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        coord_out.y++;\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize3xUp_l00_2x8;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l01_2x8;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l10_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l11_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l12_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l13_4x4;\n\
__kernel void resize_bilinear_U8toU8_SAME_3x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    coord_in.x = (short)(coord_out.x * 2 - 1) / (short)6;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
    coord_in.y = (short)(coord_out.y * 2 - 1) / (short)6;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, tmp, dst0, dst1, dst2;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in3, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l00_2x8);\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(8, 14, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l01_2x8);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l00_2x8);\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(8, 14, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l01_2x8);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_U8_half_pixel_centers_1_vx*/

static const char resize_bilinear_U8_half_pixel_centers_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l01_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l10_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l11_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l20_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l21_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l30_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l31_4x8;\n\
__kernel void resize_bilinear_U8toU8_SAME_8x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 7) >> 4;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, in2, tmp, dst0, dst1, dst2, dst3;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP4x8(dst2, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP4x8(dst2, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP4x8(dst3, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP4x8(dst3, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP4x8(dst2, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP4x8(dst2, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP4x8(dst3, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP4x8(dst3, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP4x8(dst2, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP4x8(dst2, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP4x8(dst3, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP4x8(dst3, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP4x8(dst2, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP4x8(dst2, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP4x8(dst3, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP4x8(dst3, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
    }\n\
}\n\
"; /* end of resize_bilinear_U8_half_pixel_centers_2_vx*/

static const char resize_bilinear_U8_half_pixel_centers_3_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8PostProcess_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniResize2xUp_0_4x8;\n\
_viv_uniform VXC_512Bits uniResize2xUp_1_4x8;\n\
_viv_uniform int out_height;\n\
\n\
__kernel void resize_bilinear_U8toU8_2x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 1) >> 2;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, tmp, result;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
\n\
    vxc_ushort8 dst0;\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_DP2x8(result, dst0, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        coord_out.y++;\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize4xUp_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l01_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l10_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l11_4x8;\n\
__kernel void resize_bilinear_U8toU8_4x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 3) >> 3;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, dst0, dst1;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
\n\
    vxc_ushort8 tmp;\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l00_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l10_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l10_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l10_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l10_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        coord_out.y++;\n\
    }\n\
}\n\
"; /* end of resize_bilinear_U8_half_pixel_centers_3_vx*/

static const char resize_bilinear_U8_half_pixel_centers_4_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8PostProcess_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniResize3xUp_l00_2x8;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l01_2x8;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l10_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l11_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l12_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l13_4x4;\n\
__kernel void resize_bilinear_U8toU8_3x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    coord_in.x = (short)(coord_out.x * 2 - 1) / (short)6;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
    coord_in.y = (short)(coord_out.y * 2 - 1) / (short)6;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, tmp, dst0, dst1, dst2;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in3, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
\n\
    vxc_ushort8 data;\n\
\n\
    VXC_DP4x4(data, in1, in0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(data, in1, in0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP2x8(dst2, data, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP4x4(data, in1, in0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(data, in1, in0, VXC_MODIFIER(4, 6, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_DP2x8(dst2, data, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
        VXC_MODIFIER(0, 14, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
\n\
    VXC_DP2x8(data, in1, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l00_2x8);\n\
    VXC_DP2x8(dst0, data, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP2x8(data, in1, in1, VXC_MODIFIER(0, 6, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l01_2x8);\n\
    VXC_DP2x8(dst0, data, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP4x4(data, in1, in2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(data, in1, in2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP2x8(dst1, data, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP4x4(data, in1, in2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(data, in1, in2, VXC_MODIFIER(4, 6, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_DP2x8(dst1, data, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP4x4(data, in2, in1, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(data, in2, in1, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP2x8(dst2, data, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP4x4(data, in2, in1, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(data, in2, in1, VXC_MODIFIER(4, 6, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_DP2x8(dst2, data, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
        VXC_MODIFIER(0, 14, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
        VXC_MODIFIER(0, 14, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
        VXC_MODIFIER(0, 14, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
\n\
    VXC_DP2x8(data, in2, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l00_2x8);\n\
    VXC_DP2x8(dst0, data, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP2x8(data, in2, in2, VXC_MODIFIER(0, 6, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l01_2x8);\n\
    VXC_DP2x8(dst0, data, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP4x4(data, in2, in3, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(data, in2, in3, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP2x8(dst1, data, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_DP4x4(data, in2, in3, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(data, in2, in3, VXC_MODIFIER(4, 6, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_DP2x8(dst1, data, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
        VXC_MODIFIER(0, 14, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
        VXC_MODIFIER(0, 14, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_U8_half_pixel_centers_4_vx*/

static const char resize_bilinear_U8_half_pixel_centers_5_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8PostProcess_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l01_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l10_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l11_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l20_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l21_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l30_4x8;\n\
_viv_uniform VXC_512Bits uniResize8xUp_l31_4x8;\n\
__kernel void resize_bilinear_U8toU8_8x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 7) >> 4;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, in2, dst0, dst1, dst2, dst3;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
\n\
    vxc_ushort8 tmp;\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l30_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l31_4x8);\n\
        VXC_DP2x8(dst0, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l20_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l21_4x8);\n\
        VXC_DP2x8(dst1, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l10_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l11_4x8);\n\
        VXC_DP2x8(dst2, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize8xUp_l00_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_DP4x8(tmp, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize8xUp_l01_4x8);\n\
        VXC_DP2x8(dst3, tmp, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8PostProcess_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst3,\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
    }\n\
}\n\
"; /* end of resize_bilinear_U8_half_pixel_centers_5_vx*/

static const char resize_bilinear_U8_opt_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#if (VX_VERSION==2)\n\
\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4x4_b;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_U8toU8_UP_opt\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers,\n\
    __read_only  image2d_array_t   scale\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    int4   right_x_idx = left_x_idx + 1;\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_uchar16 src0, src1;\n\
\n\
    vxc_uchar16 top_bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    vxc_ushort16 lerp_0;\n\
    vxc_half16 lerp;\n\
\n\
    int2 coord = (int2)(coord_out.x * 4, coord_out.y);\n\
    VXC_ReadImage(lerp_0.hi, scale, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lerp_0.lo, scale, coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, lerp.hi, lerp_0.hi, 16);\n\
    _viv_asm(COPY, lerp.lo, lerp_0.lo, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(top_bottom, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(top_bottom, src1, src1, maskShift, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_uchar16 dst;\n\
        VXC_DP4x4_b(dst, lerp.hi, lerp.lo, top_bottom,\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniBilinear_4x4_b);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(top_bottom, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(top_bottom, src1, src1, maskShift, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
    vxc_uchar16 dst;\n\
    VXC_DP4x4_b(dst, lerp.hi, lerp.lo, top_bottom,\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniBilinear_4x4_b);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
}\n\
\n\
#endif"; /* end of resize_bilinear_U8_opt_vx*/

static const char resize_bilinear_align_corners_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l10_4x8;\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l11_4x8;\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l20_4x8;\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l21_4x8;\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l30_4x8;\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l31_4x8;\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l40_4x8;\n\
_viv_uniform VXC_512Bits uniBilinear_8x_l41_4x8;\n\
__kernel void resize_bilinear_U8toU8_SAME_8x_upsample_align_corners\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
\n\
    vxc_uchar16 in0, in1, dst;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord.xy = coord.xy << 3;\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr);\n\
\n\
\n\
    VXC_DP4x8(dst, in0, in0, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l10_4x8);\n\
    VXC_DP4x8(dst, in0, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l11_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord.y ++;\n\
\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l10_4x8);\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l11_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord.y ++;\n\
\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l20_4x8);\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l21_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord.y ++;\n\
\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l30_4x8);\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l31_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord.y ++;\n\
\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l40_4x8);\n\
    VXC_DP4x8(dst, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l41_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord.y ++;\n\
\n\
    VXC_DP4x8(dst, in1, in0, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l30_4x8);\n\
    VXC_DP4x8(dst, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l31_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord.y ++;\n\
\n\
    VXC_DP4x8(dst, in1, in0, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l20_4x8);\n\
    VXC_DP4x8(dst, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l21_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord.y ++;\n\
\n\
    VXC_DP4x8(dst, in1, in0, VXC_MODIFIER(0, 7,  0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l10_4x8);\n\
    VXC_DP4x8(dst, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniBilinear_8x_l11_4x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst,\n\
        VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_align_corners_vx*/

static const char resize_bilinear_nhwc_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResize_x2_nhwc2_0_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x2_nhwc2_1_4x8;\n\
_viv_uniform int out_height;\n\
\n\
__kernel void resize_bilinear_nhwc_U8toU8_2x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), 0);\n\
    coord_in.x = ((coord_out.x * 2 - 1) >> 2) - 1;\n\
    coord_in.y = ((coord_out.y * 2 - 1) >> 2);\n\
    coord_in.x  = coord_out.x == 0 ? -2 : coord_in.x;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, result;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in3, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_DP4x8(result, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in2, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_DP4x8(result, in1, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in1, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_DP4x8(result, in3, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in3, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l00_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l01_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l02_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l03_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l10_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l11_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l12_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l13_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l14_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l15_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l16_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l17_4x4;\n\
__kernel void resize_bilinear_nhwc_U8toU8_3x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
    int4 coord_in   = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    coord_in.x = (short)(coord_out.x - 1) / (short)6  * 2;\n\
    coord_in.x  = coord_out.x == 0 ? -2 : coord_in.x;\n\
    coord_in.y = (short)(coord_out.y * 2 - 1) / (short)6;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, dst0, dst1, dst2;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in3, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.zw = coord_out.xy + (int2)(16, 1);\n\
\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zy, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l00_2x8);\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l01_2x8);\n\
    VXC_DP2x8(dst1, in1, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l02_2x8);\n\
    VXC_DP2x8(dst1, in1, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l03_2x8);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zw, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.yw += 2;\n\
\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zy, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zw, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.yw += 2;\n\
\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l00_2x8);\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l01_2x8);\n\
    VXC_DP2x8(dst1, in2, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l02_2x8);\n\
    VXC_DP2x8(dst1, in2, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l03_2x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zy, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zw, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l01_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l10_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l11_4x8;\n\
__kernel void resize_bilinear_nhwc_U8toU8_4x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
     )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, 0, 0);\n\
    coord_in.x = ((coord_out.x - 3) >> 3) * 2;\n\
    coord_in.y = (coord_out.y * 2 - 3) >> 3;\n\
    coord_in.x  = coord_out.x == 0 ? -2 : coord_in.x;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, dst0, dst1;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in3, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.zw = coord_out.yy + (int2)(1, 2);\n\
\n\
    VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.yz = coord_out.yz + (int2)(3, 3);\n\
\n\
    VXC_DP4x8(dst0, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst0, in2, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_DP4x8(dst1, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst1, in2, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xy, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.yw = coord_out.yw + (int2)(3, 3);\n\
\n\
    VXC_DP4x8(dst0, in1, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst0, in1, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_DP4x8(dst1, in1, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst1, in1, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_WriteImage(output, coord_out.xz, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xw, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.zw = coord_out.zw + (int2)(3, 3);\n\
\n\
    VXC_DP4x8(dst0, in3, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst0, in3, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_DP4x8(dst1, in3, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst1, in3, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}"; /* end of resize_bilinear_nhwc_vx*/

static const char resize_bilinear_nhwc_bound_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResize_x2_nhwc2_0_4x8;\n\
_viv_uniform int2 x_coord;\n\
\n\
__kernel void resize_bilinear_nhwc_bound_U8toU8_2x\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
                 image2d_array_t   output,\n\
    __write_only image2d_array_t   output1\n\
    )\n\
{\n\
    int4 coord_out =  (int4)(1, get_global_id(0), get_global_id(0), get_global_id(0));\n\
    int2 coord_in  = (int2)(1, get_global_id(0));\n\
    coord_in.y = ((coord_out.y * 2 - 1) >> 2);\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, result;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.z = coord_out.y + 1;\n\
\n\
    VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 0, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
    VXC_DP4x8(result, in2, in1, VXC_MODIFIER(0, 0, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_WriteImage(output, coord_out.xz, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.x = x_coord.x;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.x = x_coord.y;\n\
\n\
    VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 0, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
    VXC_DP4x8(result, in2, in1, VXC_MODIFIER(0, 0, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_WriteImage(output, coord_out.xz, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l10_4x4;\n\
__kernel void resize_bilinear_nhwc_bound_U8toU8_3x\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
                 image2d_array_t   output,\n\
    __write_only image2d_array_t   output1\n\
    )\n\
{\n\
    int4 coord_out =  (int4)(1, get_global_id(0), get_global_id(0), get_global_id(0));\n\
    int2 coord_in  = (int2)(1, get_global_id(0));\n\
    coord_in.y = (short)(coord_out.y * 2 - 1) / (short)6;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, result;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.zw = coord_out.yy + (int2)(1, 2);\n\
\n\
    VXC_DP4x4(result, in1, in0, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, in1, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(result, in1, in2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_WriteImage(output, coord_out.xw, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.x = x_coord.x;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.x = x_coord.y;\n\
\n\
    VXC_DP4x4(result, in1, in0, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, in1, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(result, in1, in2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_WriteImage(output, coord_out.xw, result, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l10_4x8;\n\
__kernel void resize_bilinear_nhwc_bound_U8toU8_4x\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
                 image2d_array_t   output,\n\
    __write_only image2d_array_t   output1\n\
    )\n\
{\n\
    int4 coord_out =  (int4)(1, get_global_id(0), get_global_id(0), get_global_id(0));\n\
    int2 coord_in  = (int2)(1, get_global_id(0));\n\
    coord_in.y = (coord_out.y * 2 - 3) >> 3;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, dst0, dst1;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.z = coord_out.y + 1;\n\
\n\
    VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 2;\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x -= 2;\n\
\n\
    coord_out.zw = coord_out.zz + (int2)(1, 2);\n\
    VXC_DP4x8(dst0, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst1, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_WriteImage(output, coord_out.xz, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xw, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 2;\n\
    VXC_WriteImage(output, coord_out.xz, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xw, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.x = x_coord.x;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.x = x_coord.y;\n\
    coord_out.z = coord_out.y + 1;\n\
\n\
    VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x -= 2;\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 2;\n\
\n\
    coord_out.zw = coord_out.zz + (int2)(1, 2);\n\
    VXC_DP4x8(dst0, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst1, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_WriteImage(output, coord_out.xz, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xw, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x -= 2;\n\
    VXC_WriteImage(output, coord_out.xz, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xw, dst1, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_nhwc_bound_vx*/

static const char resize_cubic_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_tail;\n\
_viv_uniform VXC_512Bits uniFp16ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniExtract8Bit_2x8;\n\
\n\
#define RESIZE_CUBIC_PART0 \\\n\
    int4   coord_out    = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4   coord_index  = coord_out; \\\n\
    int2   coord_scalew = (int2)(4 * get_global_id(0), 0); \\\n\
    int2   coord_scaleh = (int2)(4 * get_global_id(1), 0); \\\n\
    float4 cubic_coeffs_y; \\\n\
    float4 cubic_coeffs_x; \\\n\
    int4   coord_in     = (int4)(0, 0, coord_out.z, 0); \\\n\
    float4 src0_f,src1_f,src2_f,src3_f; \\\n\
    float4 dst = (float4)(0,0,0,0); \\\n\
    float  sum[4]; \\\n\
    int i = 0; \\\n\
 \\\n\
    Image scalew = create_image_from_image2d(scale_w, 4); \\\n\
    Image scaleh = create_image_from_image2d(scale_h, 4); \\\n\
 \\\n\
    uchar* scale_w_ptr = get_image_ptr_from_coord(scalew, coord_scalew); \\\n\
    __global float* scale_x = (__global float*)scale_w_ptr; \\\n\
 \\\n\
    uchar* scale_h_ptr = get_image_ptr_from_coord(scaleh, coord_scaleh); \\\n\
    __global float* scale_y = (__global float*)scale_h_ptr; \\\n\
    cubic_coeffs_y = vload4(0, scale_y); \\\n\
 \\\n\
    int index_y = read_imagei(index_h, coord_index.yw).x; \\\n\
    coord_in.y = index_y; \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
#define RESIZE_CUBIC_16Bitsto16Bits(name,src_type,dst_type,temp_type) \\\n\
__kernel void resize_cubic_##name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __read_only  image2d_t        scale_w, \\\n\
    __read_only  image2d_t        scale_h, \\\n\
    __read_only  image2d_t        index_w, \\\n\
    __read_only  image2d_t        index_h \\\n\
                           ) \\\n\
{ \\\n\
    RESIZE_CUBIC_PART0; \\\n\
    src_type     src0_h,src1_h,src2_h,src3_h; \\\n\
    vxc_short4   src0,src1,src2,src3; \\\n\
    for (i = 0; i < 4; i++) \\\n\
    { \\\n\
        coord_in.x = read_imagei(index_w, coord_index.xw).x; \\\n\
        cubic_coeffs_x = vload4(i, scale_x); \\\n\
        coord_index.x = coord_index.x + 1; \\\n\
 \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        _viv_asm(COPY, src0_h, src0, 8); \\\n\
        _viv_asm(COPY, src1_h, src1, 8); \\\n\
        _viv_asm(COPY, src2_h, src2, 8); \\\n\
        _viv_asm(COPY, src3_h, src3, 8); \\\n\
 \\\n\
        VXC_DP4x4(src0_f, src0_h, src0_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src1_f, src1_h, src1_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src2_f, src2_h, src2_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src3_f, src3_h, src3_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
 \\\n\
        dst = src0_f * cubic_coeffs_y.x \\\n\
            + src1_f * cubic_coeffs_y.y \\\n\
            + src2_f * cubic_coeffs_y.z \\\n\
            + src3_f * cubic_coeffs_y.w; \\\n\
        sum[i] = dot(dst, cubic_coeffs_x); \\\n\
    } \\\n\
    float4 sum_f = (float4)(sum[0],sum[1],sum[2],sum[3]); \\\n\
    temp_type tmpout; \\\n\
    _viv_asm(CONV,tmpout,sum_f); \\\n\
    dst_type out_h; \\\n\
    vxc_short4 out; \\\n\
    VXC_DP2x8(out_h, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8); \\\n\
    _viv_asm(COPY, out, out_h, 8); \\\n\
 \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RESIZE_CUBIC_16Bitsto16Bits(F16toF16,vxc_half4, vxc_half4, half4)\n\
RESIZE_CUBIC_16Bitsto16Bits(I16toI16,vxc_short4,vxc_short4,short4)\n\
RESIZE_CUBIC_16Bitsto16Bits(F16toI16,vxc_half4, vxc_short4,short4)\n\
RESIZE_CUBIC_16Bitsto16Bits(I16toF16,vxc_short4,vxc_half4, half4)\n\
\n\
\n\
#define RESIZE_CUBIC_Quant8toQuant8(name,data_type) \\\n\
__kernel void resize_cubic_##name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __read_only  image2d_t        scale_w, \\\n\
    __read_only  image2d_t        scale_h, \\\n\
    __read_only  image2d_t        index_w, \\\n\
    __read_only  image2d_t        index_h \\\n\
                           ) \\\n\
{ \\\n\
    RESIZE_CUBIC_PART0; \\\n\
    data_type   src0,src1,src2,src3; \\\n\
    for (i = 0; i < 4; i++) \\\n\
    { \\\n\
        coord_in.x = read_imagei(index_w, coord_index.xw).x; \\\n\
        cubic_coeffs_x = vload4(i, scale_x); \\\n\
        coord_index.x = coord_index.x + 1; \\\n\
 \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src1_f, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src2_f, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src3_f, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        src0_f = src0_f * input_scale + input_tail; \\\n\
        src1_f = src1_f * input_scale + input_tail; \\\n\
        src2_f = src2_f * input_scale + input_tail; \\\n\
        src3_f = src3_f * input_scale + input_tail; \\\n\
 \\\n\
        dst = src0_f * cubic_coeffs_y.x \\\n\
            + src1_f * cubic_coeffs_y.y \\\n\
            + src2_f * cubic_coeffs_y.z \\\n\
            + src3_f * cubic_coeffs_y.w; \\\n\
        sum[i] = dot(dst, cubic_coeffs_x); \\\n\
        sum[i] = sum[i] * output_scale + output_tail; \\\n\
    } \\\n\
    float4 sum_f = (float4)(sum[0],sum[1],sum[2],sum[3]); \\\n\
    int4 tmpout; \\\n\
    _viv_asm(CONV,tmpout,sum_f); \\\n\
    data_type out; \\\n\
    VXC_DP2x8(out, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtract8Bit_2x8); \\\n\
 \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RESIZE_CUBIC_Quant8toQuant8(U8toU8,vxc_uchar4)\n\
RESIZE_CUBIC_Quant8toQuant8(I8toI8,vxc_char4 )\n\
\n\
#define RESIZE_CUBIC_F16toQuant8(name,dst_type) \\\n\
__kernel void resize_cubic_##name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __read_only  image2d_t        scale_w, \\\n\
    __read_only  image2d_t        scale_h, \\\n\
    __read_only  image2d_t        index_w, \\\n\
    __read_only  image2d_t        index_h \\\n\
                           ) \\\n\
{ \\\n\
    RESIZE_CUBIC_PART0; \\\n\
    vxc_half4    src0_h,src1_h,src2_h,src3_h; \\\n\
    vxc_short4   src0,src1,src2,src3; \\\n\
    for (i = 0; i < 4; i++) \\\n\
    { \\\n\
        coord_in.x = read_imagei(index_w, coord_index.xw).x; \\\n\
        cubic_coeffs_x = vload4(i, scale_x); \\\n\
        coord_index.x = coord_index.x + 1; \\\n\
 \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        _viv_asm(COPY, src0_h, src0, 8); \\\n\
        _viv_asm(COPY, src1_h, src1, 8); \\\n\
        _viv_asm(COPY, src2_h, src2, 8); \\\n\
        _viv_asm(COPY, src3_h, src3, 8); \\\n\
 \\\n\
        VXC_DP4x4(src0_f, src0_h, src0_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src1_f, src1_h, src1_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src2_f, src2_h, src2_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src3_f, src3_h, src3_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
 \\\n\
        dst = src0_f * cubic_coeffs_y.x \\\n\
            + src1_f * cubic_coeffs_y.y \\\n\
            + src2_f * cubic_coeffs_y.z \\\n\
            + src3_f * cubic_coeffs_y.w; \\\n\
        sum[i] = dot(dst, cubic_coeffs_x); \\\n\
        sum[i] = sum[i] * output_scale + output_tail; \\\n\
    } \\\n\
    float4 sum_f = (float4)(sum[0],sum[1],sum[2],sum[3]); \\\n\
    int4 tmpout; \\\n\
    _viv_asm(CONV,tmpout,sum_f); \\\n\
    dst_type out; \\\n\
    VXC_DP2x8(out, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtract8Bit_2x8); \\\n\
 \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RESIZE_CUBIC_F16toQuant8(F16toU8,vxc_uchar4)\n\
RESIZE_CUBIC_F16toQuant8(F16toI8,vxc_char4)\n\
\n\
#define RESIZE_CUBIC_Quant8toF16(name,src_type) \\\n\
__kernel void resize_cubic_##name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __read_only  image2d_t        scale_w, \\\n\
    __read_only  image2d_t        scale_h, \\\n\
    __read_only  image2d_t        index_w, \\\n\
    __read_only  image2d_t        index_h \\\n\
                           ) \\\n\
{ \\\n\
    RESIZE_CUBIC_PART0; \\\n\
    src_type src0,src1,src2,src3; \\\n\
    for (i = 0; i < 4; i++) \\\n\
    { \\\n\
        coord_in.x = read_imagei(index_w, coord_index.xw).x; \\\n\
        cubic_coeffs_x = vload4(i, scale_x); \\\n\
        coord_index.x = coord_index.x + 1; \\\n\
 \\\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        VXC_DP4x4(src0_f, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src1_f, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src2_f, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(src3_f, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16ToFp32_4x4); \\\n\
 \\\n\
        src0_f = src0_f * input_scale + input_tail; \\\n\
        src1_f = src1_f * input_scale + input_tail; \\\n\
        src2_f = src2_f * input_scale + input_tail; \\\n\
        src3_f = src3_f * input_scale + input_tail; \\\n\
 \\\n\
        dst = src0_f * cubic_coeffs_y.x \\\n\
            + src1_f * cubic_coeffs_y.y \\\n\
            + src2_f * cubic_coeffs_y.z \\\n\
            + src3_f * cubic_coeffs_y.w; \\\n\
        sum[i] = dot(dst, cubic_coeffs_x); \\\n\
    } \\\n\
    float4 sum_f = (float4)(sum[0],sum[1],sum[2],sum[3]); \\\n\
    half4 tmpout; \\\n\
    _viv_asm(CONV,tmpout,sum_f); \\\n\
    vxc_half4 out_h; \\\n\
    vxc_short4 out; \\\n\
    VXC_DP2x8(out_h, tmpout, tmpout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8); \\\n\
    _viv_asm(COPY, out, out_h, 8); \\\n\
 \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, out, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
RESIZE_CUBIC_Quant8toF16(U8toF16,vxc_uchar4)\n\
RESIZE_CUBIC_Quant8toF16(I8toF16,vxc_char4)"; /* end of resize_cubic_vx*/

static const char resize_nearest_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniMultiplyAndPostShift_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform float round_value;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4   coord_x    = coord_out.xxxx + (int4)(0, 1, 2, 3); \\\n\
    float4 in_x       = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx + round_value; \\\n\
    int4   in_x_idx   = convert_int4(in_x); \\\n\
    float  in_y       = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y + round_value; \\\n\
    int    in_y_idx   = convert_int(in_y); \\\n\
\n\
\n\
__kernel void resize_nearest_F16toF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniGetExtractData_2x8;\n\
__kernel void resize_nearest_F16toF16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_2x8;\n\
__kernel void resize_nearest_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_char16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_I8toI8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst0;\n\
    vxc_char16 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_U8toU8\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(src, src, multiplier, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_U8toU8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(dst, dst, multiplier, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_I16toI16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_I16toI16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_short8 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //in_x_idx = in_x_idx - in_x_idx.xxxx;\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_nearest_vx*/

static const char rope_0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale0;\n\
_viv_uniform float scale1;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int half_head_size;\n\
_viv_uniform VXC_512Bits uniATimesB_0_4x4;\n\
_viv_uniform VXC_512Bits uniATimesB_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define ROPE_BNHS_SYMM(name, src_type, src1_type, copy_type, dst_type) \\\n\
__kernel void rope_##name##_bnhs \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord_in; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    src_type data0, data1; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
    dst_type dst; \\\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v0, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
    coord_in.y += half_head_size; \\\n\
    VXC_OP4(img_load_3d, data1, input, coord_in.xywz, 0, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    float4 data2, data3, data4, data5; \\\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data3, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data5, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    data2 = data2 * scale0 - data4 * scale1 + output_zp; \\\n\
    data3 = data3 * scale0 - data5 * scale1 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
                dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(data2, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    VXC_DP4x4(data4, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    data2 = data2 * scale1 + data4 * scale0 + output_zp; \\\n\
    data3 = data3 * scale1 + data5 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    coord_out.y += half_head_size; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
            dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_BNHS_SYMM(I16_I16toI16, vxc_short8, vxc_short8, vxc_short8, vxc_short8)\n\
ROPE_BNHS_SYMM(I16_I16toI8,  vxc_short8, vxc_short8, vxc_short8, vxc_char8)\n\
ROPE_BNHS_SYMM(I16_I16toU8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8)\n\
ROPE_BNHS_SYMM(I16_F16toI16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
ROPE_BNHS_SYMM(I16_F16toI8,  vxc_short8, vxc_half8,  vxc_short8, vxc_char8)\n\
ROPE_BNHS_SYMM(I16_F16toU8,  vxc_short8, vxc_half8,  vxc_short8, vxc_uchar8)\n\
\n\
__kernel void rope_F16_F16toF16_bnhs\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord_in;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    vxc_short8 v0, v1, v2, v3, dst;\n\
    vxc_half8 data0, data1, cos, sin, dst2;\n\
    VXC_OP4(img_load_3d, v0, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, v0, 16);\n\
    VXC_ReadImage(v1, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, cos, v1, 16);\n\
    VXC_ReadImage(v2, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, sin, v2, 16);\n\
    coord_in.y += half_head_size;\n\
    VXC_OP4(img_load_3d, v3, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, v3, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 data2, data3, data4, data5;\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data3, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data5, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    data2 = data2 - data4;\n\
    data3 = data3 - data5;\n\
\n\
    half4 dst0;\n\
    half4 dst1;\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(data2, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    VXC_DP4x4(data4, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    data2 = data2 * scale1 + data4 * scale0 + output_zp;\n\
    data3 = data3 * scale1 + data5 * scale0 + output_zp;\n\
\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    coord_out.y += half_head_size;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform int in0_zp;\n\
_viv_uniform int cos_zp;\n\
_viv_uniform int sin_zp;\n\
_viv_uniform VXC_512Bits uniAMinusZp_0_4x4;\n\
_viv_uniform VXC_512Bits uniAMinusZp_1_4x4;\n\
#define ROPE_ASYM_BNHS(name, src1_type, copy_type, dtype) \\\n\
__kernel void rope_##name##_bnhs \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4 coord_out = coord_in; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    dtype data0, data1, dst; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v0, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
    coord_in.y += half_head_size; \\\n\
    VXC_OP4(img_load_3d, data1, input, coord_in.xywz, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    float4 l00, l01, cos0, cos1; \\\n\
    float4 l10, l11, sin0, sin1; \\\n\
    VXC_DP4x4(l00, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(l01, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(cos0, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(cos1, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(sin0, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(sin1, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(l10, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(l11, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    float4 data2 = l00 * cos0 * scale0 - l10 * sin0 * scale1 + output_zp; \\\n\
    float4 data3 = l01 * cos1 * scale0 - l11 * sin1 * scale1 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, \\\n\
            coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    data2 = l00 * sin0 * scale1 + l10 * cos0 * scale0 + output_zp; \\\n\
    data3 = l01 * sin1 * scale1 + l11 * cos1 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    coord_out.y += half_head_size; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, \\\n\
        coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_ASYM_BNHS(I8_I8toI8,    vxc_char8,   vxc_char8,   vxc_char8)\n\
ROPE_ASYM_BNHS(U8_U8toU8,    vxc_uchar8,  vxc_uchar8,  vxc_uchar8)\n\
ROPE_ASYM_BNHS(U16_U16toU16, vxc_ushort8, vxc_ushort8, vxc_ushort8)\n\
ROPE_ASYM_BNHS(I8_F16toI8,   vxc_half8,   vxc_short8,  vxc_char8)\n\
ROPE_ASYM_BNHS(U8_F16toU8,   vxc_half8,   vxc_short8,  vxc_uchar8)\n\
ROPE_ASYM_BNHS(U16_F16toU16, vxc_half8,   vxc_short8,  vxc_ushort8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
__kernel void rope_BF16_BF16toBF16_bnhs\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord_in;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    vxc_ushort8 v0, v1, v2, v3, dst;\n\
    VXC_OP4(img_load_3d, v0, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(v1, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(v2, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y += half_head_size;\n\
    VXC_OP4(img_load_3d, v3, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 src0, src1, src2, src3, cos0, cos1, sin0, sin1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_short8 data;\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src0, data, 16);\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src1, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, cos0, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, cos1, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, sin0, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, sin1, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src2, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src3, data, 16);\n\
\n\
    float4 data0 = src0 * cos0 - src2 * sin0;\n\
    float4 data1 = src1 * cos1 - src3 * sin1;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
    data0 = src0 * sin0 + src2 * cos0;\n\
    data1 = src1 * sin1 + src3 * cos1;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    coord_out.y += half_head_size;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of rope_0_vx*/

static const char rope_1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale0;\n\
_viv_uniform float scale1;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int half_head_size;\n\
_viv_uniform VXC_512Bits uniATimesB_0_4x4;\n\
_viv_uniform VXC_512Bits uniATimesB_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define ROPE_BNH1_SYMM(name, src_type, src1_type, copy_type, dst_type) \\\n\
__kernel void rope_##name##_bnh1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type data0, data1; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
    VXC_ReadImage(data0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v0, cos_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
    coord.x += half_head_size; \\\n\
    VXC_ReadImage(data1, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 data2, data3, data4, data5; \\\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data3, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data5, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    data2 = data2 * scale0 - data4 * scale1 + output_zp; \\\n\
    data3 = data3 * scale0 - data5 * scale1 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    dst_type dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord.zw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(data2, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    VXC_DP4x4(data4, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4); \\\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4); \\\n\
    data2 = data2 * scale1 + data4 * scale0 + output_zp; \\\n\
    data3 = data3 * scale1 + data5 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_BNH1_SYMM(I16_I16toI16, vxc_short8, vxc_short8, vxc_short8, vxc_short8)\n\
ROPE_BNH1_SYMM(I16_I16toI8,  vxc_short8, vxc_short8, vxc_short8, vxc_char8)\n\
ROPE_BNH1_SYMM(I16_I16toU8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8)\n\
ROPE_BNH1_SYMM(I16_F16toI16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
ROPE_BNH1_SYMM(I16_F16toI8,  vxc_short8, vxc_half8,  vxc_short8, vxc_char8)\n\
ROPE_BNH1_SYMM(I16_F16toU8,  vxc_short8, vxc_half8,  vxc_short8, vxc_uchar8)\n\
\n\
__kernel void rope_F16_F16toF16_bnh1\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 v0, v1, v2, v3, dst;\n\
    vxc_half8 data0, data1, cos, sin, dst2;\n\
    VXC_ReadImage(v0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, v0, 16);\n\
    VXC_ReadImage(v1, cos_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, cos, v1, 16);\n\
    VXC_ReadImage(v2, sin_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, sin, v2, 16);\n\
    coord.x += half_head_size;\n\
    VXC_ReadImage(v3, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, v3, 16);\n\
\n\
    float4 data2, data3, data4, data5;\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data3, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data5, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    data2 = data2 - data4;\n\
    data3 = data3 - data5;\n\
\n\
    half4 dst0;\n\
    half4 dst1;\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    VXC_WriteImage(output, coord.zw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(data2, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    VXC_DP4x4(data4, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_0_4x4);\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniATimesB_1_4x4);\n\
    data2 = data2 + data4;\n\
    data3 = data3 + data5;\n\
\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform int in0_zp;\n\
_viv_uniform int cos_zp;\n\
_viv_uniform int sin_zp;\n\
_viv_uniform VXC_512Bits uniAMinusZp_0_4x4;\n\
_viv_uniform VXC_512Bits uniAMinusZp_1_4x4;\n\
#define ROPE_ASYM_BNH1(name, src1_type, copy_type, dtype) \\\n\
__kernel void rope_##name##_bnh1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    dtype data0, data1, dst; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
    VXC_ReadImage(data0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(v0, cos_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
    coord.x += half_head_size; \\\n\
    VXC_ReadImage(data1, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 l00, l01, cos0, cos1; \\\n\
    float4 l10, l11, sin0, sin1; \\\n\
    VXC_DP4x4(l00, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(l01, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(cos0, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(cos1, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(sin0, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(sin1, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(l10, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(l11, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    float4 data2 = l00 * cos0 * scale0 - l10 * sin0 * scale1 + output_zp; \\\n\
    float4 data3 = l01 * cos1 * scale0 - l11 * sin1 * scale1 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord.zw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    data2 = l00 * sin0 * scale1 + l10 * cos0 * scale0 + output_zp; \\\n\
    data3 = l01 * sin1 * scale1 + l11 * cos1 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_ASYM_BNH1(I8_I8toI8,    vxc_char8,   vxc_char8,   vxc_char8)\n\
ROPE_ASYM_BNH1(U8_U8toU8,    vxc_uchar8,  vxc_uchar8,  vxc_uchar8)\n\
ROPE_ASYM_BNH1(U16_U16toU16, vxc_ushort8, vxc_ushort8, vxc_ushort8)\n\
ROPE_ASYM_BNH1(I8_F16toI8,   vxc_half8,   vxc_short8,  vxc_char8)\n\
ROPE_ASYM_BNH1(U8_F16toU8,   vxc_half8,   vxc_short8,  vxc_uchar8)\n\
ROPE_ASYM_BNH1(U16_F16toU16, vxc_half8,   vxc_short8,  vxc_ushort8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
__kernel void rope_BF16_BF16toBF16_bnh1\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    vxc_ushort8 v0, v1, v2, v3, dst;\n\
    VXC_ReadImage(v0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(v1, cos_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(v2, sin_cache, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.x += half_head_size;\n\
    VXC_ReadImage(v3, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 src0, src1, src2, src3, cos0, cos1, sin0, sin1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_short8 data;\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src0, data, 16);\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src1, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, cos0, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, cos1, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, sin0, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, sin1, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src2, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src3, data, 16);\n\
\n\
    float4 data0 = src0 * cos0 - src2 * sin0;\n\
    float4 data1 = src1 * cos1 - src3 * sin1;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord.zw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    data0 = src0 * sin0 + src2 * cos0;\n\
    data1 = src1 * sin1 + src3 * cos1;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of rope_1_vx*/

static const char rope_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale0;\n\
_viv_uniform float scale1;\n\
_viv_uniform float output_zp;\n\
_viv_uniform VXC_512Bits uniAEvenTimesB_0_4x4;\n\
_viv_uniform VXC_512Bits uniAEvenTimesB_1_4x4;\n\
_viv_uniform VXC_512Bits uniAOddTimesB_0_4x4;\n\
_viv_uniform VXC_512Bits uniAOddTimesB_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define ROPE_BSNH_SYMM(name, src_type, src1_type, copy_type, dst_type) \\\n\
__kernel void rope_##name##_bsnh \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    src_type data0, data1; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
    dst_type dst; \\\n\
    VXC_ReadImage(v0, cos_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
 \\\n\
    coord_in.x *= 2; \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, data1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord_in; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    float4 data2, data3, data4, data5; \\\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4); \\\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4); \\\n\
    VXC_DP4x4(data4, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4); \\\n\
    VXC_DP4x4(data5, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4); \\\n\
    data2 = data2 * scale0 - data4 * scale1 + output_zp; \\\n\
    data3 = data3 * scale1 + data5 * scale0 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
                dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(data2, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4); \\\n\
    VXC_DP4x4(data3, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4); \\\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4); \\\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4); \\\n\
    data2 = data2 * scale0 - data4 * scale1 + output_zp; \\\n\
    data3 = data3 * scale1 + data5 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    coord_out.x += 8; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
            dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_BSNH_SYMM(I16_I16toI16, vxc_short8, vxc_short8, vxc_short8, vxc_short8)\n\
ROPE_BSNH_SYMM(I16_I16toI8,  vxc_short8, vxc_short8, vxc_short8, vxc_char8)\n\
ROPE_BSNH_SYMM(I16_I16toU8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8)\n\
ROPE_BSNH_SYMM(I16_F16toI16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
ROPE_BSNH_SYMM(I16_F16toI8,  vxc_short8, vxc_half8,  vxc_short8, vxc_char8)\n\
ROPE_BSNH_SYMM(I16_F16toU8,  vxc_short8, vxc_half8,  vxc_short8, vxc_uchar8)\n\
\n\
__kernel void rope_F16_F16toF16_bsnh\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    vxc_short8 v0, v1, v2, v3, dst;\n\
    vxc_half8 data0, data1, cos, sin, dst2;\n\
    VXC_ReadImage(v1, cos_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, cos, v1, 16);\n\
    VXC_ReadImage(v2, sin_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, sin, v2, 16);\n\
\n\
    coord_in.x *= 2;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, v0, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, v0, 16);\n\
    VXC_OP4(img_load_3d, v3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, v3, 16);\n\
\n\
    int4 coord_out = coord_in;\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 data2, data3, data4, data5;\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4);\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4);\n\
    VXC_DP4x4(data4, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4);\n\
    VXC_DP4x4(data5, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4);\n\
    data2 = data2 - data4;\n\
    data3 = data3 + data5;\n\
\n\
    half4 dst0;\n\
    half4 dst1;\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(data2, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4);\n\
    VXC_DP4x4(data3, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4);\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4);\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4);\n\
    data2 = data2 - data4;\n\
    data3 = data3 + data5;\n\
\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    coord_out.x += 8;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform int in0_zp;\n\
_viv_uniform int cos_zp;\n\
_viv_uniform int sin_zp;\n\
_viv_uniform VXC_512Bits uniAMinusZp_0_4x4;\n\
_viv_uniform VXC_512Bits uniAMinusZp_1_4x4;\n\
_viv_uniform VXC_512Bits uniAEvenMinusZp_4x4;\n\
_viv_uniform VXC_512Bits uniAOddMinusZp_4x4;\n\
#define ROPE_ASYM_BSNH(name, src1_type, copy_type, dtype) \\\n\
__kernel void rope_##name##_bsnh \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    dtype data0, data1, dst; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
 \\\n\
    VXC_ReadImage(v0, cos_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
    coord_in.x *= 2; \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, data1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord_in; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    float4 l00, l01, cos0, cos1; \\\n\
    float4 l10, l11, sin0, sin1; \\\n\
    VXC_DP4x4(l00, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenMinusZp_4x4); \\\n\
    VXC_DP4x4(l01, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddMinusZp_4x4); \\\n\
    VXC_DP4x4(cos0, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(cos1, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(sin0, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(sin1, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    float4 data2 = l00 * cos0 * scale0 - l01 * sin0 * scale1 + output_zp; \\\n\
    float4 data3 = l00 * sin0 * scale1 + l01 * cos0 * scale0 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, \\\n\
            coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(l10, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenMinusZp_4x4); \\\n\
    VXC_DP4x4(l11, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddMinusZp_4x4); \\\n\
    data2 = l10 * cos1 * scale0 - l11 * sin1 * scale1 + output_zp; \\\n\
    data3 = l10 * sin1 * scale1 + l11 * cos1 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    coord_out.x += 8; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, \\\n\
        coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_ASYM_BSNH(I8_I8toI8,    vxc_char8,   vxc_char8,   vxc_char8)\n\
ROPE_ASYM_BSNH(U8_U8toU8,    vxc_uchar8,  vxc_uchar8,  vxc_uchar8)\n\
ROPE_ASYM_BSNH(U16_U16toU16, vxc_ushort8, vxc_ushort8, vxc_ushort8)\n\
ROPE_ASYM_BSNH(I8_F16toI8,   vxc_half8,   vxc_short8,  vxc_char8)\n\
ROPE_ASYM_BSNH(U8_F16toU8,   vxc_half8,   vxc_short8,  vxc_uchar8)\n\
ROPE_ASYM_BSNH(U16_F16toU16, vxc_half8,   vxc_short8,  vxc_ushort8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
__kernel void rope_BF16_BF16toBF16_bsnh\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    vxc_ushort8 v0, v1, v2, v3, dst;\n\
    VXC_ReadImage(v1, cos_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(v2, sin_cache, coord_in.xw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.x *= 2;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, v0, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, v3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int4 coord_out = coord_in;\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 src0, src1, src2, src3, cos0, cos1, sin0, sin1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_short8 data;\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src0, data, 16);\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src1, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, cos0, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, cos1, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, sin0, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, sin1, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src2, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src3, data, 16);\n\
\n\
    float4 even = (float4)(src0.xz, src1.xz);\n\
    float4 odd = (float4)(src0.yw, src1.yw);\n\
    float4 data0 = even * cos0 - odd * sin0;\n\
    float4 data1 = even * sin0 + odd * cos0;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
    even = (float4)(src2.xz, src3.xz);\n\
    odd = (float4)(src2.yw, src3.yw);\n\
    data0 = even * cos1 - odd * sin1;\n\
    data1 = even * sin1 + odd * cos1;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    coord_out.x += 8;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of rope_2_vx*/

static const char rope_3_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale0;\n\
_viv_uniform float scale1;\n\
_viv_uniform float output_zp;\n\
_viv_uniform VXC_512Bits uniAEvenTimesB_0_4x4;\n\
_viv_uniform VXC_512Bits uniAEvenTimesB_1_4x4;\n\
_viv_uniform VXC_512Bits uniAOddTimesB_0_4x4;\n\
_viv_uniform VXC_512Bits uniAOddTimesB_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define ROPE_BNSH_SYMM(name, src_type, src1_type, copy_type, dst_type) \\\n\
__kernel void rope_##name##_bnsh \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    src_type data0, data1; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
    dst_type dst; \\\n\
    VXC_ReadImage(v0, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
 \\\n\
    coord_in.x *= 2; \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, data1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord_in; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    float4 data2, data3, data4, data5; \\\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4); \\\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4); \\\n\
    VXC_DP4x4(data4, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4); \\\n\
    VXC_DP4x4(data5, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4); \\\n\
    data2 = data2 * scale0 - data4 * scale1 + output_zp; \\\n\
    data3 = data3 * scale1 + data5 * scale0 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
                dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(data2, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4); \\\n\
    VXC_DP4x4(data3, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4); \\\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4); \\\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4); \\\n\
    data2 = data2 * scale0 - data4 * scale1 + output_zp; \\\n\
    data3 = data3 * scale1 + data5 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    coord_out.x += 8; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, \\\n\
            dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_BNSH_SYMM(I16_I16toI16, vxc_short8, vxc_short8, vxc_short8, vxc_short8)\n\
ROPE_BNSH_SYMM(I16_I16toI8,  vxc_short8, vxc_short8, vxc_short8, vxc_char8)\n\
ROPE_BNSH_SYMM(I16_I16toU8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8)\n\
ROPE_BNSH_SYMM(I16_F16toI16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
ROPE_BNSH_SYMM(I16_F16toI8,  vxc_short8, vxc_half8,  vxc_short8, vxc_char8)\n\
ROPE_BNSH_SYMM(I16_F16toU8,  vxc_short8, vxc_half8,  vxc_short8, vxc_uchar8)\n\
\n\
__kernel void rope_F16_F16toF16_bnsh\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    vxc_short8 v0, v1, v2, v3, dst;\n\
    vxc_half8 data0, data1, cos, sin, dst2;\n\
    VXC_ReadImage(v1, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, cos, v1, 16);\n\
    VXC_ReadImage(v2, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, sin, v2, 16);\n\
\n\
    coord_in.x *= 2;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, v0, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, v0, 16);\n\
    VXC_OP4(img_load_3d, v3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, v3, 16);\n\
\n\
    int4 coord_out = coord_in;\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 data2, data3, data4, data5;\n\
    VXC_DP4x4(data2, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4);\n\
    VXC_DP4x4(data3, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_0_4x4);\n\
    VXC_DP4x4(data4, data0, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4);\n\
    VXC_DP4x4(data5, data0, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_0_4x4);\n\
    data2 = data2 - data4;\n\
    data3 = data3 + data5;\n\
\n\
    half4 dst0;\n\
    half4 dst1;\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(data2, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4);\n\
    VXC_DP4x4(data3, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenTimesB_1_4x4);\n\
    VXC_DP4x4(data4, data1, sin, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4);\n\
    VXC_DP4x4(data5, data1, cos, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddTimesB_1_4x4);\n\
    data2 = data2 - data4;\n\
    data3 = data3 + data5;\n\
\n\
    _viv_asm(CONV_RTE, dst0, data2);\n\
    _viv_asm(CONV_RTE, dst1, data3);\n\
\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst2, 16);\n\
    coord_out.x += 8;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform int in0_zp;\n\
_viv_uniform int cos_zp;\n\
_viv_uniform int sin_zp;\n\
_viv_uniform VXC_512Bits uniAMinusZp_0_4x4;\n\
_viv_uniform VXC_512Bits uniAMinusZp_1_4x4;\n\
_viv_uniform VXC_512Bits uniAEvenMinusZp_4x4;\n\
_viv_uniform VXC_512Bits uniAOddMinusZp_4x4;\n\
#define ROPE_ASYM_BNSH(name, src1_type, copy_type, dtype) \\\n\
__kernel void rope_##name##_bnsh \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t cos_cache, \\\n\
    __read_only  image2d_array_t sin_cache, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int   axis \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    dtype data0, data1, dst; \\\n\
    src1_type cos, sin; \\\n\
    copy_type v0, v1; \\\n\
 \\\n\
    VXC_ReadImage(v0, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, cos, v0, 16); \\\n\
    VXC_ReadImage(v1, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, sin, v1, 16); \\\n\
    coord_in.x *= 2; \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
    VXC_OP4(img_load_3d, data0, input, coord_in.xywz, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, data1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0), \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int4 coord_out = coord_in; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    float4 l00, l01, cos0, cos1; \\\n\
    float4 l10, l11, sin0, sin1; \\\n\
    VXC_DP4x4(l00, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenMinusZp_4x4); \\\n\
    VXC_DP4x4(l01, data0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddMinusZp_4x4); \\\n\
    VXC_DP4x4(cos0, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(cos1, cos, cos_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    VXC_DP4x4(sin0, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_0_4x4); \\\n\
    VXC_DP4x4(sin1, sin, sin_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAMinusZp_1_4x4); \\\n\
    float4 data2 = l00 * cos0 * scale0 - l01 * sin0 * scale1 + output_zp; \\\n\
    float4 data3 = l00 * sin0 * scale1 + l01 * cos0 * scale0 + output_zp; \\\n\
 \\\n\
    int4 dst0 = convert_int4_rte(data2); \\\n\
    int4 dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, \\\n\
            coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(l10, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAEvenMinusZp_4x4); \\\n\
    VXC_DP4x4(l11, data1, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAOddMinusZp_4x4); \\\n\
    data2 = l10 * cos1 * scale0 - l11 * sin1 * scale1 + output_zp; \\\n\
    data3 = l10 * sin1 * scale1 + l11 * cos1 * scale0 + output_zp; \\\n\
 \\\n\
    dst0 = convert_int4_rte(data2); \\\n\
    dst1 = convert_int4_rte(data3); \\\n\
 \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    coord_out.x += 8; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, \\\n\
        coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
ROPE_ASYM_BNSH(I8_I8toI8,    vxc_char8,   vxc_char8,   vxc_char8)\n\
ROPE_ASYM_BNSH(U8_U8toU8,    vxc_uchar8,  vxc_uchar8,  vxc_uchar8)\n\
ROPE_ASYM_BNSH(U16_U16toU16, vxc_ushort8, vxc_ushort8, vxc_ushort8)\n\
ROPE_ASYM_BNSH(I8_F16toI8,   vxc_half8,   vxc_short8,  vxc_char8)\n\
ROPE_ASYM_BNSH(U8_F16toU8,   vxc_half8,   vxc_short8,  vxc_uchar8)\n\
ROPE_ASYM_BNSH(U16_F16toU16, vxc_half8,   vxc_short8,  vxc_ushort8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
__kernel void rope_BF16_BF16toBF16_bnsh\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int   axis\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    vxc_ushort8 v0, v1, v2, v3, dst;\n\
    VXC_ReadImage(v1, cos_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(v2, sin_cache, coord_in.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.x *= 2;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, v0, input, coord_in.xywz, 0,\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, v3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int4 coord_out = coord_in;\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 src0, src1, src2, src3, cos0, cos1, sin0, sin1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_short8 data;\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src0, data, 16);\n\
    VXC_DP2x8(data, v0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src1, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, cos0, data, 16);\n\
    VXC_DP2x8(data, v1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, cos1, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, sin0, data, 16);\n\
    VXC_DP2x8(data, v2, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, sin1, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, src2, data, 16);\n\
    VXC_DP2x8(data, v3, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, src3, data, 16);\n\
\n\
    float4 even = (float4)(src0.xz, src1.xz);\n\
    float4 odd = (float4)(src0.yw, src1.yw);\n\
    float4 data0 = even * cos0 - odd * sin0;\n\
    float4 data1 = even * sin0 + odd * cos0;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
    even = (float4)(src2.xz, src3.xz);\n\
    odd = (float4)(src2.yw, src3.yw);\n\
    data0 = even * cos1 - odd * sin1;\n\
    data1 = even * sin1 + odd * cos1;\n\
\n\
    _viv_asm(COPY, v0, data0, 16);\n\
    _viv_asm(COPY, v1, data1, 16);\n\
\n\
    VXC_DP2x8(dst, v0, v1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    coord_out.x += 8;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of rope_3_vx*/

static const char scatter_nd_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int zeropoint;\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void scatter_nd_F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            VXC_ReadImage(tmpVal, input1, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            _viv_asm(COPY, src, tmpVal, 16);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    _viv_asm(COPY, tmpVal, sum, 16);\n\
    VXC_WriteImage(output, (int2)(gidx, gidy), tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void scatter_nd_BF16toBF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    vxc_ushort8 zero = (vxc_ushort8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 sum0 = (float4)(0);\n\
    float4 sum1 = (float4)(0);\n\
    vxc_ushort8 tmpVal;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_ushort8 src0, src1;\n\
            VXC_ReadImage(tmpVal, input1, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            float4 vec0, vec1;\n\
            VXC_DP2x8(src0, tmpVal, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
            _viv_asm(COPY, vec0, src0, 16);\n\
            VXC_DP2x8(src1, tmpVal, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, vec1, src1, 16);\n\
            sum0 += vec0;\n\
            sum1 += vec1;\n\
        }\n\
    }\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    _viv_asm(COPY, dst0, sum0, 16);\n\
    _viv_asm(COPY, dst1, sum1, 16);\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, (int2)(gidx, gidy), dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define SCATTER_ND_QINT(src0_type_name, data_type) \\\n\
__kernel void scatter_nd_##src0_type_name##to##src0_type_name##( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    image2d_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0);  \\\n\
    int gidy = get_global_id(1); \\\n\
    int firstFlg = 1; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = read_imagei(input0, (int2)(0, i)); \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type src; \\\n\
            VXC_ReadImage(src, input1, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
            if(firstFlg) \\\n\
            { \\\n\
                firstFlg = 0; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    if(firstFlg) \\\n\
    { \\\n\
        sum = (data_type)(zeropoint, zeropoint, zeropoint, zeropoint, \\\n\
                             zeropoint, zeropoint, zeropoint, zeropoint); \\\n\
    } \\\n\
    VXC_WriteImage(output, (int2)(gidx, gidy), sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SCATTER_ND_QINT(U8,  vxc_uchar8)\n\
SCATTER_ND_QINT(I8,  vxc_char8)\n\
SCATTER_ND_QINT(I16, vxc_short8)\n\
"; /* end of scatter_nd_vx*/

static const char scatter_nd_big_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
_viv_uniform int zeropoint;\n\
\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
\n\
__kernel void scatter_nd_F16toF16_big(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image i0_img = create_image_from_image2d(input0, 4);\n\
    __global int* index_ptr = (__global int*)i0_img.ptr;\n\
    Image i1_img = create_image_from_image2d(input1, 2);\n\
    __global short* update_ptr = (__global short*)i1_img.ptr;\n\
    Image o_img = create_image_from_image2d(output, 2);\n\
    __global short* output_ptr = (__global short*)o_img.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = vload4(0, index_ptr + i * coord_dim);\n\
\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            short tmpData = update_ptr[i * update_width + gidx];\n\
            _viv_asm(COPY, src, tmpData, 4);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    short dst;\n\
    _viv_asm(COPY, dst, sum, 4);\n\
    output_ptr[gidy * output_width+ gidx] = dst;\n\
}\n\
\n\
#define SCATTER_ND_QINT_BIG(src0_type_name, data_type, ptr_type) \\\n\
__kernel void scatter_nd_##src0_type_name##to##src0_type_name##_big( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    image2d_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0);  \\\n\
    int gidy = get_global_id(1); \\\n\
    int firstFlg = 1; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    Image i0_img = create_image_from_image2d(input0, 2); \\\n\
    __global int* index_ptr = (__global int*)i0_img.ptr; \\\n\
    Image i1_img = create_image_from_image2d(input1, 2); \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)i1_img.ptr; \\\n\
    Image o_img = create_image_from_image2d(output, 2); \\\n\
    __global ptr_type* output_ptr = (__global ptr_type*)o_img.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + i * coord_dim); \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type src; \\\n\
            ptr_type tmpData = update_ptr[i * update_width + gidx]; \\\n\
            _viv_asm(COPY, src, tmpData, 4); \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
            if(firstFlg) \\\n\
            { \\\n\
                firstFlg = 0; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    ptr_type dst; \\\n\
    if(firstFlg) \\\n\
    { \\\n\
        _viv_asm(COPY, dst, zeropoint, 4); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        _viv_asm(COPY, dst, sum, 4); \\\n\
    } \\\n\
    output_ptr[gidy * output_width+ gidx] = dst; \\\n\
}\n\
SCATTER_ND_QINT_BIG(U8,  vxc_uchar8, uchar)\n\
SCATTER_ND_QINT_BIG(I8,  vxc_char8, char)\n\
SCATTER_ND_QINT_BIG(I16, vxc_short8, short)\n\
"; /* end of scatter_nd_big_vx*/

static const char scatter_nd_update_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_1_Lo_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int offset_idx;\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
_viv_uniform int offsetW;\n\
_viv_uniform int2 multAndoutZP0;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void scatter_nd_update_F16F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_array_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        //int4 indice = read_imagei(input1, (int2)(0, i));\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            VXC_ReadImage(tmpVal, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpVal, 16);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    _viv_asm(COPY, tmpVal, sum, 16);\n\
    int2 coord = (int2)(gidx, gidy);\n\
    if(cnt == 0)\n\
    {\n\
        VXC_ReadImage(tmpVal, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    VXC_WriteImage(output, coord, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void scatter_nd_update_F16F16toU8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_array_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        //int4 indice = read_imagei(input1, (int2)(0, i));\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            VXC_ReadImage(tmpVal, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpVal, 16);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_ushort8 ms0;\n\
    vxc_uchar8 dst;\n\
    if(cnt == 0)\n\
    {\n\
        vxc_half8 src;\n\
        VXC_ReadImage(tmpVal, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
        _viv_asm(COPY, src, tmpVal, 16);\n\
        VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_0_Lo_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    else\n\
    {\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16);\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_1_Lo_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define SCATTER_ND_UPDATE_QINT(src0_type_name, src2_type_name, out_type_name, data_type) \\\n\
__kernel void scatter_nd_update_##src0_type_name##src2_type_name##to##out_type_name##( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    image2d_array_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0);  \\\n\
    int gidy = get_global_id(1); \\\n\
    int cnt = 0; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + offset_idx); \\\n\
        index_ptr += coord_dim; \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type src; \\\n\
            VXC_ReadImage(src, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            cnt++; \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
        } \\\n\
    } \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    vxc_ushort8 ms0; \\\n\
    data_type dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        VXC_ReadImage(sum, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_1_Lo_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT(U8,  U8,  U8,  vxc_uchar8)\n\
SCATTER_ND_UPDATE_QINT(I8,  I8,  I8,  vxc_char8)\n\
SCATTER_ND_UPDATE_QINT(I16, I16, I16, vxc_short8)\n\
\n\
#define SCATTER_ND_UPDATE_QINT_TO_F16(src0_type, data_type) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##toF16( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    image2d_array_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int cnt = 0; \\\n\
    vxc_short8 sum = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    data_type src; \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + offset_idx); \\\n\
        index_ptr += coord_dim; \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            VXC_ReadImage(src, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            cnt++; \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
        } \\\n\
    } \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    vxc_ushort8 ms0; \\\n\
    vxc_half8 tmpDst; \\\n\
    vxc_short8 dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        VXC_ReadImage(src, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        VXC_DP2x8(tmpDst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, 16); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16); \\\n\
        VXC_DP2x8(tmpDst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_1_Lo_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, 16); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_TO_F16(U8,  vxc_uchar8)\n\
SCATTER_ND_UPDATE_QINT_TO_F16(I8,  vxc_char8)\n\
SCATTER_ND_UPDATE_QINT_TO_F16(I16, vxc_short8)\n\
\n\
__kernel void scatter_nd_update_BF16BF16toBF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_array_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_ushort8 src0, src1, src2;\n\
    float4 srcA, srcB;\n\
    float4 sum0 = (float4)(0);\n\
    float4 sum1 = sum0;\n\
\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        //int4 indice = read_imagei(input1, (int2)(0, i));\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            VXC_ReadImage(src0, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            cnt++;\n\
            VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
            VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, srcA, src1, 16);\n\
            _viv_asm(COPY, srcB, src2, 16);\n\
            sum0 += srcA;\n\
            sum1 += srcB;\n\
        }\n\
    }\n\
    int2 coord = (int2)(gidx, gidy);\n\
    if(cnt == 0)\n\
    {\n\
        VXC_ReadImage(src2, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    else\n\
    {\n\
        _viv_asm(COPY, src0, sum0, 16);\n\
        _viv_asm(COPY, src1, sum1, 16);\n\
        VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of scatter_nd_update_vx*/

static const char scatter_nd_update_atom_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
_viv_uniform int2 multAndoutZP0;\n\
\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
_viv_uniform int offsetW;\n\
_viv_uniform int offset_idx;\n\
\n\
_viv_uniform float scaleInOut;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float input_scale;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform int count_width;\n\
\n\
#define SCATTER_ND_UPDATE_QINT_PRE(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_##src0_type##_pre( \\\n\
    __read_only image2d_t   input1,  __read_only image2d_t   input2, \\\n\
    image2d_t  output, image2d_t  output_cnt, image2d_t tmp_output, \\\n\
    int width, int area, int vol, int coord_dim ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    Image img2 = create_image_from_image2d(input2, element_size); \\\n\
    Image img3 = create_image_from_image2d(output, 4); \\\n\
    Image img4 = create_image_from_image2d(output_cnt, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global int* output_ptr = (__global int*)img3.ptr; \\\n\
    __global int* cnt_ptr = (__global int*)img4.ptr; \\\n\
    data_type src; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim + offset_idx); \\\n\
    ptr_type tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    _viv_asm(COPY, src, tmpData, 4); \\\n\
    vxc_int4 data; \\\n\
    short zp = input_zp; \\\n\
    VXC_DP4x4(data, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
             uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    atomic_add(output_ptr + loc, data.x); \\\n\
    if(gidx == 0) \\\n\
    { \\\n\
        atomic_inc(cnt_ptr + idx); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_PRE(U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_QINT_PRE(I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_QINT_PRE(I16, vxc_short8, short, 2)\n\
\n\
// input0  ref\n\
// input1  sum\n\
// input2  count\n\
// input3  update\n\
#define SCATTER_ND_UPDATE_QINT_TO_F16_BIG(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##toF16_big( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    __read_only image2d_t   input3, \\\n\
    __read_only image2d_t   input4, \\\n\
    image2d_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    Image img2 = create_image_from_image2d(input2, 4); \\\n\
    Image img3 = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    __global int* cnt_ptr = (__global int*)img2.ptr; \\\n\
    __global short* output_ptr = (__global short*)img3.ptr; \\\n\
    data_type src; \\\n\
 \\\n\
    int cnt = cnt_ptr[gidy]; \\\n\
    int loc = gidy * output_width + gidx; \\\n\
 \\\n\
    vxc_ushort8 ms0; \\\n\
    vxc_half8 tmpDst; \\\n\
    vxc_short8 dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        Image img0 = create_image_from_image2d(input0, element_size); \\\n\
        __global ptr_type* ref_ptr = (__global ptr_type*)img0.ptr; \\\n\
        ptr_type tmpData = ref_ptr[loc]; \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        _viv_asm(COPY, src, tmpData, 4); \\\n\
        VXC_DP2x8(tmpDst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, 16); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        Image img1 = create_image_from_image2d(input1, 4); \\\n\
        __global int* sum_ptr = (__global int*)img1.ptr; \\\n\
        int sum = sum_ptr[loc]; \\\n\
        float result = sum * input_scale; \\\n\
        half tmpOut; \\\n\
        _viv_asm(CONV, tmpOut, result); \\\n\
        _viv_asm(COPY, dst, tmpOut, 4); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_TO_F16_BIG(U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_QINT_TO_F16_BIG(I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_QINT_TO_F16_BIG(I16, vxc_short8, short, 2)\n\
\n\
#define SCATTER_ND_UPDATE_QINT_BIG(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##to##src0_type##_big( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    __read_only image2d_t   input3, \\\n\
    __read_only image2d_t   input4, \\\n\
    image2d_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img2 = create_image_from_image2d(input2, 4); \\\n\
    Image img3 = create_image_from_image2d(output, element_size); \\\n\
    __global int* cnt_ptr = (__global int*)img2.ptr; \\\n\
    __global ptr_type* output_ptr = (__global ptr_type*)img3.ptr; \\\n\
    int cnt = cnt_ptr[gidy]; \\\n\
    int loc = gidy * output_width + gidx; \\\n\
    data_type src, dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        Image img0 = create_image_from_image2d(input0, element_size); \\\n\
        __global ptr_type* ref_ptr = (__global ptr_type*)img0.ptr; \\\n\
        ptr_type tmpData = ref_ptr[loc]; \\\n\
        vxc_ushort8 ms0; \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        _viv_asm(COPY, src, tmpData, 4); \\\n\
        VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        Image img1 = create_image_from_image2d(input1, 4); \\\n\
        __global int* sum_ptr = (__global int*)img1.ptr; \\\n\
        int sum = sum_ptr[loc]; \\\n\
        int4 result; \\\n\
        result.x = convert_int_rte(sum * scaleInOut + output_zp); \\\n\
        VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_BIG(U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_QINT_BIG(I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_QINT_BIG(I16, vxc_short8, short, 2)\n\
\n\
__kernel void scatter_nd_update_reset(\n\
    __read_only image2d_t   input0,\n\
    image2d_t  output_sum,\n\
    image2d_t  output_cnt\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
\n\
    Image img3 = create_image_from_image2d(output_sum, 4);\n\
    Image img4 = create_image_from_image2d(output_cnt, 4);\n\
    __global int* sum_ptr = (__global int*)img3.ptr;\n\
    __global int* cnt_ptr = (__global int*)img4.ptr;\n\
    int4 data = (int4)(0);\n\
    vstore4(data, gidx, sum_ptr);\n\
    if(gidx < count_width)\n\
    {\n\
        vstore4(data, gidx, cnt_ptr);\n\
    }\n\
}"; /* end of scatter_nd_update_atom_vx*/

static const char scatter_nd_update_big_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
_viv_uniform int offsetW;\n\
_viv_uniform int offset_idx;\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_1_Lo_2x8;\n\
_viv_uniform int2 multAndoutZP0;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
__kernel void scatter_nd_update_F16F16toF16_big(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 2);\n\
    Image img3 = create_image_from_image2d(output, 2);\n\
\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global short* update_ptr = (__global short*)img2.ptr;\n\
    __global short* output_ptr = (__global short*)img3.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            short tmpData = update_ptr[i * update_width + gidx];\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpData, 4);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    short dst;\n\
    _viv_asm(COPY, dst, sum, 4);\n\
    int loc = gidy * output_width+ gidx;\n\
    if(cnt == 0)\n\
    {\n\
        Image img0 = create_image_from_image2d(input0, 2);\n\
        __global short* ref_ptr = (__global short*)img0.ptr;\n\
        dst = ref_ptr[loc];\n\
    }\n\
    output_ptr[loc] = dst;\n\
}\n\
\n\
__kernel void scatter_nd_update_F16F16toU8_big(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 2);\n\
    Image img3 = create_image_from_image2d(output, 1);\n\
\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global short* update_ptr = (__global short*)img2.ptr;\n\
    __global uchar* output_ptr = (__global uchar*)img3.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            short tmpData = update_ptr[i * update_width + gidx];\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpData, 4);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    short dst;\n\
    vxc_ushort8 ms0;\n\
    int loc = gidy * output_width+ gidx;\n\
    if(cnt == 0)\n\
    {\n\
        vxc_half8 src;\n\
        Image img0 = create_image_from_image2d(input0, 2);\n\
        __global short* ref_ptr = (__global short*)img0.ptr;\n\
        short tmpData = ref_ptr[loc];\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
        _viv_asm(COPY, src, tmpData, 4);\n\
        VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_0_Lo_2x8);\n\
        output_ptr[loc] = dst;\n\
    }\n\
    else\n\
    {\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16);\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_1_Lo_2x8);\n\
        output_ptr[loc] = dst;\n\
    }\n\
}\n\
"; /* end of scatter_nd_update_big_vx*/

static const char scatter_nd_update_fp_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
_viv_uniform int ref_stride;\n\
_viv_uniform int output_stride;\n\
\n\
_viv_uniform int4 coord_stride;\n\
_viv_uniform int4 coord_stride1;\n\
_viv_uniform float inout_scale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16ToFp32_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
inline void AtomicAdd_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = prevVal.floatVal + operand;\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
__kernel void scatter_nd_update_update_F16(\n\
    __read_only image2d_t   index,\n\
    __read_only image2d_t   update,\n\
    image2d_t  temp_buf_float,\n\
    image2d_t  link_buffer0,\n\
    int width, int area, int vol, int val4,\n\
    int val5, int val6, int val7, int coord_dim)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    Image img1 = create_image_from_image2d(index, 4);\n\
    Image img2 = create_image_from_image2d(update, 2);\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global short* update_ptr = (__global short*)img2.ptr;\n\
    __global float* output_ptr = (__global float*)img3.ptr;\n\
    half src;\n\
\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim);\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim);\n\
    short tmpData = update_ptr[gidy * update_width + gidx];\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1;\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w;\n\
    int loc = idx * output_width + gidx;\n\
    _viv_asm(COPY, src, tmpData, 4);\n\
    float data;\n\
    _viv_asm(CONV, data, src);\n\
    AtomicAdd_float(output_ptr + loc, data);\n\
}\n\
\n\
__kernel void scatter_nd_update_update_F16_4X(\n\
    __read_only image2d_t   index,\n\
    __read_only image2d_t   update,\n\
    image2d_t  temp_buf_float,\n\
    image2d_t  link_buffer0,\n\
    int width, int area, int vol, int val4,\n\
    int val5, int val6, int val7, int coord_dim)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    Image img1 = create_image_from_image2d(index, 4);\n\
    Image img2 = create_image_from_image2d(update, 2);\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global vxc_short4* update_ptr = (__global vxc_short4*)img2.ptr;\n\
    __global float* output_ptr = (__global float*)img3.ptr;\n\
    vxc_half4 src;\n\
\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim);\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim);\n\
    vxc_short4 tmpData = update_ptr[gidy * update_width + gidx];\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1;\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w;\n\
    int4 loc = idx * output_width + gidx * 4 + (int4)(0, 1, 2, 3);\n\
\n\
    _viv_asm(COPY, src, tmpData, 8);\n\
    float4 data;\n\
    VXC_DP4x4(data, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\n\
             uniConvertFp16ToFp32_4x4);\n\
    AtomicAdd_float(output_ptr + loc.x, data.x);\n\
    AtomicAdd_float(output_ptr + loc.y, data.y);\n\
    AtomicAdd_float(output_ptr + loc.z, data.z);\n\
    AtomicAdd_float(output_ptr + loc.w, data.w);\n\
}\n\
\n\
__kernel void scatter_nd_update_update_BF16(\n\
    __read_only image2d_t   index,\n\
    __read_only image2d_t   update,\n\
    image2d_t  temp_buf_float,\n\
    image2d_t  link_buffer0,\n\
    int width, int area, int vol, int val4,\n\
    int val5, int val6, int val7, int coord_dim)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    Image img1 = create_image_from_image2d(index, 4);\n\
    Image img2 = create_image_from_image2d(update, 2);\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global short* update_ptr = (__global short*)img2.ptr;\n\
    __global float* output_ptr = (__global float*)img3.ptr;\n\
    float data;\n\
\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim);\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim);\n\
    short tmpData = update_ptr[gidy * update_width + gidx];\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_short8 src0, src1;\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1;\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w;\n\
    int loc = idx * output_width + gidx;\n\
    _viv_asm(COPY, src0, tmpData, 4);\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, data, src1, 4);\n\
    AtomicAdd_float(output_ptr + loc, data);\n\
}\n\
\n\
__kernel void scatter_nd_update_update_BF16_4X(\n\
    __read_only image2d_t   index,\n\
    __read_only image2d_t   update,\n\
    image2d_t  temp_buf_float,\n\
    image2d_t  link_buffer0,\n\
    int width, int area, int vol, int val4,\n\
    int val5, int val6, int val7, int coord_dim)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    Image img1 = create_image_from_image2d(index, 4);\n\
    Image img2 = create_image_from_image2d(update, 2);\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global vxc_short4* update_ptr = (__global vxc_short4*)img2.ptr;\n\
    __global float* output_ptr = (__global float*)img3.ptr;\n\
\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim);\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim);\n\
    vxc_short4 tmpData = update_ptr[gidy * update_width + gidx];\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_short8 src0, src1;\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1;\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w;\n\
    int4 loc = idx * output_width + gidx * 4 + (int4)(0, 1, 2, 3);\n\
\n\
    _viv_asm(COPY, src0, tmpData, 8);\n\
    float4 data;\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, data, src1, 16);\n\
    AtomicAdd_float(output_ptr + loc.x, data.x);\n\
    AtomicAdd_float(output_ptr + loc.y, data.y);\n\
    AtomicAdd_float(output_ptr + loc.z, data.z);\n\
    AtomicAdd_float(output_ptr + loc.w, data.w);\n\
}\n\
\n\
#define SCATTER_ND_UPDATE_REF_FP16(type0, type1, ptr_type) \\\n\
__kernel void scatter_nd_update_ref_##type0##to##type1( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    __read_only image2d_t   temp_buf_int, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  link_buffer0, \\\n\
    image2d_t  link_buffer1, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(temp_buf_int, 4); \\\n\
    Image img3 = create_image_from_image2d(temp_ref, 2); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* acc_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global short* ref_ptr = (__global short*)img3.ptr; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    int loc = idx * output_stride + gidx; \\\n\
    float4 tmpData; \\\n\
    tmpData.x = convert_float(acc_ptr[loc]) * inout_scale + output_zp; \\\n\
    half4 data; \\\n\
    short tmpDst; \\\n\
    _viv_asm(CONV, data, tmpData); \\\n\
    _viv_asm(COPY, tmpDst, data, 4); \\\n\
    ref_ptr[loc] = tmpDst; \\\n\
}\n\
SCATTER_ND_UPDATE_REF_FP16(I32, F16, int)\n\
SCATTER_ND_UPDATE_REF_FP16(F32, F16, float)\n\
\n\
#define SCATTER_ND_UPDATE_REF_FP16_4X(type0, type1, ptr_type) \\\n\
__kernel void scatter_nd_update_ref_##type0##to##type1##_4X( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    __read_only image2d_t   temp_buf_int, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  link_buffer0, \\\n\
    image2d_t  link_buffer1, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(temp_buf_int, 4); \\\n\
    Image img3 = create_image_from_image2d(temp_ref, 2); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* acc_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global vxc_short4* ref_ptr = (__global vxc_short4*)img3.ptr; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    float4 tmpData = convert_float4(vload4(gidx, acc_ptr + idx * ref_stride)); \\\n\
    int loc = idx * output_stride + gidx; \\\n\
    float4 tmpVal = tmpData * inout_scale + output_zp; \\\n\
    half4 data; \\\n\
    vxc_short8 tmpDst; \\\n\
    _viv_asm(CONV, data, tmpVal); \\\n\
    _viv_asm(COPY, tmpDst, data, 16); \\\n\
    ref_ptr[loc] = tmpDst.s0246; \\\n\
}\n\
SCATTER_ND_UPDATE_REF_FP16_4X(I32, F16, int)\n\
SCATTER_ND_UPDATE_REF_FP16_4X(F32, F16, float)\n\
\n\
__kernel void scatter_nd_update_ref_F32toBF16(\n\
    __read_only image2d_t   index,\n\
    __read_only image2d_t   update,\n\
    __read_only image2d_t   temp_buf_int,\n\
    image2d_t  temp_ref,\n\
    image2d_t  link_buffer0,\n\
    image2d_t  link_buffer1,\n\
    int width, int area, int vol, int val4,\n\
    int val5, int val6, int val7, int coord_dim)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    Image img1 = create_image_from_image2d(index, 4);\n\
    Image img2 = create_image_from_image2d(temp_buf_int, 4);\n\
    Image img3 = create_image_from_image2d(temp_ref, 2);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global float* acc_ptr = (__global float*)img2.ptr;\n\
    __global short* ref_ptr = (__global short*)img3.ptr;\n\
\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim);\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim);\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1;\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w;\n\
    int loc = idx * output_stride + gidx;\n\
    float tmpData;\n\
    tmpData = acc_ptr[loc];\n\
    vxc_ushort8 src0, src2;\n\
    _viv_asm(COPY, src0, tmpData, 4);\n\
    VXC_DP2x8(src2, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
    ref_ptr[loc] = src2.x;\n\
}\n\
\n\
__kernel void scatter_nd_update_ref_F32toBF16_4X(\n\
    __read_only image2d_t   index,\n\
    __read_only image2d_t   update,\n\
    __read_only image2d_t   temp_buf_int,\n\
    image2d_t  temp_ref,\n\
    image2d_t  link_buffer0,\n\
    image2d_t  link_buffer1,\n\
    int width, int area, int vol, int val4,\n\
    int val5, int val6, int val7, int coord_dim)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    Image img1 = create_image_from_image2d(index, 4);\n\
    Image img2 = create_image_from_image2d(temp_buf_int, 4);\n\
    Image img3 = create_image_from_image2d(temp_ref, 2);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global float* acc_ptr = (__global float*)img2.ptr;\n\
    __global vxc_short4* ref_ptr = (__global vxc_short4*)img3.ptr;\n\
\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim);\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim);\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1;\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w;\n\
    float4 tmpData = vload4(gidx, acc_ptr + idx * ref_stride);\n\
    int loc = idx * output_stride + gidx;\n\
    vxc_short8 src0, src2;\n\
    _viv_asm(COPY, src0, tmpData, 16);\n\
    VXC_DP2x8(src2, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    ref_ptr[loc] = src2.s0123;\n\
}\n\
"; /* end of scatter_nd_update_fp_vx*/

static const char scatter_nd_update_qint_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
_viv_uniform int ref_stride;\n\
_viv_uniform int output_stride;\n\
_viv_uniform int2 multAndoutZP0;\n\
\n\
_viv_uniform int4 coord_stride;\n\
_viv_uniform int4 coord_stride1;\n\
\n\
_viv_uniform float output_zp;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float inout_scale;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
#define SCATTER_RESET(name0, name1, ptr0, ptr1, type0, type1, len0, len1, size0, size1, ptr2, ptr3, len3) \\\n\
__kernel void scatter_nd_update_reset_##name0##to##name1( \\\n\
    __read_only image2d_t   input_ref, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  temp_buf_int, \\\n\
    int length, int res) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img1 = create_image_from_image2d(input_ref, size0); \\\n\
    Image img2 = create_image_from_image2d(temp_ref, size1); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_int, 4); \\\n\
    __global int* tmp_update_ptr = (__global int*)img3.ptr; \\\n\
    type0 src; \\\n\
    type1 tmpDst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    if(length > 0) \\\n\
    { \\\n\
        __global ptr0* input_ptr = (__global ptr0*)img1.ptr; \\\n\
        __global ptr1* output_ptr = (__global ptr1*)img2.ptr; \\\n\
        ptr0 tmpData = input_ptr[gidx]; \\\n\
        int4 zeros = (int4)(0); \\\n\
        int loc2 = gidx * 8; \\\n\
        ptr1 dst; \\\n\
        _viv_asm(COPY, src, tmpData, len0); \\\n\
        VXC_DP2x8(tmpDst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, len1); \\\n\
        output_ptr[gidx] = dst; \\\n\
        vstore4(zeros, 0, tmp_update_ptr + loc2); \\\n\
        vstore4(zeros, 1, tmp_update_ptr + loc2); \\\n\
    } \\\n\
    __global ptr2* input_ptr1 = (__global ptr2*)img1.ptr; \\\n\
    __global ptr3* output_ptr1 = (__global ptr3*)img2.ptr; \\\n\
    for(int i = gidx; i < res; i += get_global_size(0)) \\\n\
    { \\\n\
        ptr2 tmpData1 = input_ptr1[length + i]; \\\n\
        ptr3 dst1; \\\n\
        dst1 ^= dst1; \\\n\
        tmp_update_ptr[length + i] = 0; \\\n\
        _viv_asm(COPY, src, tmpData1, 4); \\\n\
        VXC_DP2x8(tmpDst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        _viv_asm(COPY, dst1, tmpDst, len3); \\\n\
        output_ptr1[length + i] = dst1; \\\n\
    } \\\n\
}\n\
SCATTER_RESET(U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, 8, 8, 1, 1, uchar, uchar, 1)\n\
SCATTER_RESET(I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  8, 8, 1, 1, char, char, 1)\n\
SCATTER_RESET(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, 16, 16, 2, 2, short, short, 2)\n\
SCATTER_RESET(F16, F16, vxc_short8, vxc_short8, vxc_half8,  vxc_half8,  16, 16, 2, 2, short, short, 2)\n\
SCATTER_RESET(U8,  F16, vxc_uchar8, vxc_short8, vxc_uchar8, vxc_half8,  8, 16, 1, 2, uchar, short, 2)\n\
SCATTER_RESET(I8,  F16, vxc_char8,  vxc_short8, vxc_char8,  vxc_half8,  8, 16, 1, 2, char, short, 2)\n\
SCATTER_RESET(I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  16, 8, 2, 1, short, short, 2)\n\
SCATTER_RESET(F16, U8,  vxc_short8, vxc_uchar8, vxc_half8,  vxc_uchar8, 16, 8, 2, 1, short, uchar, 1)\n\
\n\
__kernel void scatter_nd_update_reset_BF16toBF16(\n\
    __read_only image2d_t   input_ref,\n\
    image2d_t  temp_ref,\n\
    image2d_t  temp_buf_int)\n\
{\n\
    int gidx = get_global_id(0);\n\
    Image img1 = create_image_from_image2d(input_ref, 2);\n\
    Image img2 = create_image_from_image2d(temp_ref, 2);\n\
    Image img3 = create_image_from_image2d(temp_buf_int, 4);\n\
    __global vxc_short8* input_ptr = (__global vxc_short8*)img1.ptr;\n\
    __global vxc_short8* output_ptr = (__global vxc_short8*)img2.ptr;\n\
    __global float* tmp_update_ptr = (__global float*)img3.ptr;\n\
    vxc_short8 src = input_ptr[gidx];\n\
    float4 zeros = (float4)(0, 0, 0, 0);\n\
    int loc2 = gidx * 8;\n\
    output_ptr[gidx] = src;\n\
    vstore4(zeros, 0, tmp_update_ptr + loc2);\n\
    vstore4(zeros, 1, tmp_update_ptr + loc2);\n\
}\n\
\n\
#define SCATTER_ND_UPDATE_QINT(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_update_##src0_type( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    image2d_t  temp_buf_int, \\\n\
    image2d_t  link_buffer0, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(update, element_size); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_int, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global int* output_ptr = (__global int*)img3.ptr; \\\n\
    data_type src; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    ptr_type tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    _viv_asm(COPY, src, tmpData, 4); \\\n\
    vxc_int4 data; \\\n\
    short zp = input_zp; \\\n\
    VXC_DP4x4(data, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
             uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    atomic_add(output_ptr + loc, data.x); \\\n\
}\n\
SCATTER_ND_UPDATE_QINT(U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_QINT(I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_QINT(I16, vxc_short8, short, 2)\n\
\n\
#define SCATTER_ND_UPDATE_QINT_4X(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_update_##src0_type##_4X( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    image2d_t  temp_buf_int, \\\n\
    image2d_t  link_buffer0, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(update, element_size); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_int, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global int* output_ptr = (__global int*)img3.ptr; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    ptr_type src = update_ptr[gidy * update_width + gidx]; \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    int4 loc = idx * output_width + gidx * 4 + (int4)(0, 1, 2, 3); \\\n\
    vxc_int4 data; \\\n\
    short zp = input_zp; \\\n\
    VXC_DP4x4(data, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
             uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    atomic_add(output_ptr + loc.x, data.x); \\\n\
    atomic_add(output_ptr + loc.y, data.y); \\\n\
    atomic_add(output_ptr + loc.z, data.z); \\\n\
    atomic_add(output_ptr + loc.w, data.w); \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_4X(U8,  vxc_uchar8, vxc_uchar4, 1)\n\
SCATTER_ND_UPDATE_QINT_4X(I8,  vxc_char8,  vxc_char4,  1)\n\
SCATTER_ND_UPDATE_QINT_4X(I16, vxc_short8, vxc_short4, 2)\n\
\n\
#define SCATTER_ND_UPDATE_REF(src0_type, dst_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_ref_##src0_type##to##dst_type( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    __read_only image2d_t   temp_buf_int, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  link_buffer0, \\\n\
    image2d_t  link_buffer1, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(temp_buf_int, 4); \\\n\
    Image img3 = create_image_from_image2d(temp_ref, element_size); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global int* acc_ptr = (__global int*)img2.ptr; \\\n\
    __global ptr_type* ref_ptr = (__global ptr_type*)img3.ptr; \\\n\
    data_type dst; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    int loc = idx * output_stride + gidx; \\\n\
    int tmpData = acc_ptr[loc]; \\\n\
    int4 data; \\\n\
    data.x = convert_int_rte(tmpData * inout_scale + output_zp); \\\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
    ref_ptr[loc] = dst.x; \\\n\
}\n\
SCATTER_ND_UPDATE_REF(I32, U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_REF(I32, I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_REF(I32, I16, vxc_short8, short, 2)\n\
\n\
#define SCATTER_ND_UPDATE_REF_4X(src0_type, dst_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_ref_##src0_type##to##dst_type##_4X( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    __read_only image2d_t   temp_buf_int, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  link_buffer0, \\\n\
    image2d_t  link_buffer1, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(temp_buf_int, 4); \\\n\
    Image img3 = create_image_from_image2d(temp_ref, element_size); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global int* acc_ptr = (__global int*)img2.ptr; \\\n\
    __global ptr_type* ref_ptr = (__global ptr_type*)img3.ptr; \\\n\
    data_type dst; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    float4 tmpData = convert_float4(vload4(gidx, acc_ptr + idx * ref_stride)); \\\n\
    int loc = idx * output_stride + gidx; \\\n\
    int4 data = convert_int4_rte(tmpData * inout_scale + output_zp); \\\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
    ref_ptr[loc] = dst.xyzw; \\\n\
}\n\
SCATTER_ND_UPDATE_REF_4X(I32, U8,  vxc_uchar8, vxc_uchar4, 1)\n\
SCATTER_ND_UPDATE_REF_4X(I32, I8,  vxc_char8,  vxc_char4,  1)\n\
SCATTER_ND_UPDATE_REF_4X(I32, I16, vxc_short8, vxc_short4, 2)\n\
\n\
#define SCATTER_ND_UPDATE_COPY(src0_type, ptr_type, element_size, ptr_type1) \\\n\
__kernel void scatter_nd_update_copy_##src0_type( \\\n\
    __read_only image2d_t  temp_ref, \\\n\
    __read_only image2d_t  link_buffer1, \\\n\
    image2d_t  output, \\\n\
    int length, int res) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img1 = create_image_from_image2d(temp_ref, element_size); \\\n\
    Image img2 = create_image_from_image2d(output, element_size); \\\n\
    if(length > 0) \\\n\
    { \\\n\
        __global ptr_type* input_ptr = (__global ptr_type*)img1.ptr; \\\n\
        __global ptr_type* output_ptr = (__global ptr_type*)img2.ptr; \\\n\
        output_ptr[gidx] = input_ptr[gidx]; \\\n\
    } \\\n\
    __global ptr_type1* input_ptr1 = (__global ptr_type1*)img1.ptr; \\\n\
    __global ptr_type1* output_ptr1 = (__global ptr_type1*)img2.ptr; \\\n\
    for(int i = gidx; i < res; i += get_global_size(0)) \\\n\
    { \\\n\
        output_ptr1[length + i] = input_ptr1[length + i]; \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_COPY(U8,  vxc_uchar8, 1, uchar)\n\
SCATTER_ND_UPDATE_COPY(I8,  vxc_char8,  1, char)\n\
SCATTER_ND_UPDATE_COPY(I16, vxc_short8, 2, short)\n\
SCATTER_ND_UPDATE_COPY(F16, vxc_short8, 2, short)\n\
SCATTER_ND_UPDATE_COPY(BF16, vxc_short8, 2, short)\n\
"; /* end of scatter_nd_update_qint_vx*/

static const char scatter_nd_update_reduction_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
\n\
_viv_uniform int4 coord_stride;\n\
_viv_uniform int4 coord_stride1;\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform int input_zp;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int update_zp;\n\
_viv_uniform float update_scale;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndU8SubZpToFp32_4x4;\n\
\n\
inline void AtomicAdd_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = prevVal.floatVal + operand;\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
inline void AtomicMul_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = prevVal.floatVal * operand;\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
inline void AtomicMax_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = fmax(prevVal.floatVal, operand);\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
inline void AtomicMin_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = fmin(prevVal.floatVal, operand);\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
#define SCATTER_REDUCTION_PREPROCESS(name0, ptr0, type0, len0, size0, ptr2) \\\n\
__kernel void scatter_nd_update_reduction_preprocess_##name0( \\\n\
    __read_only image2d_t   input_ref, \\\n\
    image2d_t  temp_buf_float, \\\n\
    int length, int res) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img1 = create_image_from_image2d(input_ref, size0); \\\n\
    Image img2 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    __global float* tmp_ref_ptr = (__global float*)img2.ptr; \\\n\
    type0 src; \\\n\
    float4 tmpDst0, tmpDst1; \\\n\
    short zp = input_zp; \\\n\
    if(length > 0) \\\n\
    { \\\n\
        __global ptr0* input_ptr = (__global ptr0*)img1.ptr; \\\n\
        ptr0 tmpData = input_ptr[gidx]; \\\n\
        int loc2 = gidx * 8; \\\n\
        _viv_asm(COPY, src, tmpData, len0); \\\n\
        VXC_DP4x4(tmpDst0, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tmpDst1, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvert2ndU8SubZpToFp32_4x4); \\\n\
        tmpDst0 *= input_scale; \\\n\
        tmpDst1 *= input_scale; \\\n\
        vstore4(tmpDst0, 0, tmp_ref_ptr + loc2); \\\n\
        vstore4(tmpDst1, 1, tmp_ref_ptr + loc2); \\\n\
    } \\\n\
    __global ptr2* input_ptr1 = (__global ptr2*)img1.ptr; \\\n\
    for(int i = gidx; i < res; i += get_global_size(0)) \\\n\
    { \\\n\
        ptr2 tmpData1 = input_ptr1[length + i]; \\\n\
        _viv_asm(COPY, src, tmpData1, 4); \\\n\
        VXC_DP4x4(tmpDst0, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        tmp_ref_ptr[length + i] = tmpDst0.x; \\\n\
    } \\\n\
}\n\
SCATTER_REDUCTION_PREPROCESS(U8,  vxc_uchar8, vxc_uchar8, 8,  1, uchar)\n\
SCATTER_REDUCTION_PREPROCESS(I8,  vxc_char8,  vxc_char8,  8,  1, char)\n\
SCATTER_REDUCTION_PREPROCESS(I16, vxc_short8, vxc_short8, 16, 2, short)\n\
SCATTER_REDUCTION_PREPROCESS(F16, vxc_short8, vxc_half8,  16, 2, short)\n\
\n\
#define SCATTER_ND_REDUCTION_PROCESS_F16(name0, func) \\\n\
__kernel void scatter_nd_update_reduction_##name0##_F16( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    image2d_t  temp_buf_float, \\\n\
    image2d_t  link_buffer0, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(update, 2); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global short* update_ptr = (__global short*)img2.ptr; \\\n\
    __global float* output_ptr = (__global float*)img3.ptr; \\\n\
    half src; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    short tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    _viv_asm(COPY, src, tmpData, 4); \\\n\
    float data; \\\n\
    _viv_asm(CONV, data, src); \\\n\
    func(output_ptr + loc, data); \\\n\
}\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Add,  AtomicAdd_float)\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Mul,  AtomicMul_float)\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Max,  AtomicMax_float)\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Min,  AtomicMin_float)\n\
\n\
#define SCATTER_ND_REDUCTION_PROCESS_BF16(name0, func) \\\n\
__kernel void scatter_nd_update_reduction_##name0##_BF16( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    image2d_t  temp_buf_float, \\\n\
    image2d_t  link_buffer0, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(update, 2); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global short* update_ptr = (__global short*)img2.ptr; \\\n\
    __global float* output_ptr = (__global float*)img3.ptr; \\\n\
    half src; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    short tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_short8 src0, src1; \\\n\
    float data; \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    _viv_asm(COPY, src0, tmpData, 4); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                        uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, data, src1, 4); \\\n\
    func(output_ptr + loc, data); \\\n\
}\n\
SCATTER_ND_REDUCTION_PROCESS_BF16(Add,  AtomicAdd_float)\n\
SCATTER_ND_REDUCTION_PROCESS_BF16(Mul,  AtomicMul_float)\n\
SCATTER_ND_REDUCTION_PROCESS_BF16(Max,  AtomicMax_float)\n\
SCATTER_ND_REDUCTION_PROCESS_BF16(Min,  AtomicMin_float)\n\
\n\
#define SCATTER_ND_UPDATE_PROCESS_QINT(name0, src0_type, data_type, ptr_type, element_size, func) \\\n\
__kernel void scatter_nd_update_reduction_##name0##_##src0_type( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    image2d_t  temp_buf_float, \\\n\
    image2d_t  link_buffer0, \\\n\
    int width, int area, int vol, int val4, \\\n\
    int val5, int val6, int val7, int coord_dim) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(update, element_size); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global float* output_ptr = (__global float*)img3.ptr; \\\n\
    data_type src; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    ptr_type tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    short zp = update_zp; \\\n\
    int4 tmpOffset = indice * coord_stride + indice1 * coord_stride1; \\\n\
    int idx = tmpOffset.x + tmpOffset.y + tmpOffset.z + tmpOffset.w; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    _viv_asm(COPY, src, tmpData, 4); \\\n\
    vxc_float4 data; \\\n\
    VXC_DP4x4(data, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    data.x *= update_scale; \\\n\
    func(output_ptr + loc, data.x); \\\n\
}\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Add, U8,  vxc_uchar8, uchar, 1, AtomicAdd_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Mul, U8,  vxc_uchar8, uchar, 1, AtomicMul_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Max, U8,  vxc_uchar8, uchar, 1, AtomicMax_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Min, U8,  vxc_uchar8, uchar, 1, AtomicMin_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Add, I8,  vxc_char8,  char,  1, AtomicAdd_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Mul, I8,  vxc_char8,  char,  1, AtomicMul_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Max, I8,  vxc_char8,  char,  1, AtomicMax_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Min, I8,  vxc_char8,  char,  1, AtomicMin_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Add, I16, vxc_short8, short, 2, AtomicAdd_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Mul, I16, vxc_short8, short, 2, AtomicMul_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Max, I16, vxc_short8, short, 2, AtomicMax_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Min, I16, vxc_short8, short, 2, AtomicMin_float)\n\
"; /* end of scatter_nd_update_reduction_vx*/

static const char scatter_nd_update_reduction_conv_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
\n\
#define SCATTER_ND_UPDATE_CONV(src0_type, ptr_type, element_size, ptr_type1, conv_func) \\\n\
__kernel void scatter_nd_update_reduction_conv_##src0_type( \\\n\
    __read_only image2d_t  temp_buf_float, \\\n\
    __read_only image2d_t  link_buf, \\\n\
    image2d_t  output, \\\n\
    int length, int res) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img1 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    Image img2 = create_image_from_image2d(output, element_size); \\\n\
    __global float* input_ptr = (__global float*)img1.ptr; \\\n\
    if(length > 0) \\\n\
    { \\\n\
        __global ptr_type* output_ptr = (__global ptr_type*)img2.ptr; \\\n\
        float4 src0 = vload4(0, input_ptr + gidx * 8); \\\n\
        float4 src1 = vload4(1, input_ptr + gidx * 8); \\\n\
        int4 data0 = convert_int4_rte(src0 * output_scale + output_zp); \\\n\
        int4 data1 = convert_int4_rte(src1 * output_scale + output_zp); \\\n\
        ptr_type dst; \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
        output_ptr[gidx] = dst; \\\n\
    } \\\n\
    __global ptr_type1* output_ptr1 = (__global ptr_type1*)img2.ptr; \\\n\
    for(int i = gidx; i < res; i += get_global_size(0)) \\\n\
    { \\\n\
        float src = input_ptr[length + i]; \\\n\
        int data = convert_int_rte(src * output_scale + output_zp); \\\n\
        output_ptr1[length + i] = conv_func(data); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_CONV(U8,  vxc_uchar8, 1, uchar, convert_uchar)\n\
SCATTER_ND_UPDATE_CONV(I8,  vxc_char8,  1, char,  convert_char)\n\
SCATTER_ND_UPDATE_CONV(I16, vxc_short8, 2, short, convert_short)\n\
\n\
__kernel void scatter_nd_update_reduction_conv_F16(\n\
    __read_only image2d_t  temp_buf_float,\n\
    __read_only image2d_t  link_buf,\n\
    image2d_t  output,\n\
    int length, int res)\n\
{\n\
    int gidx = get_global_id(0);\n\
    Image img1 = create_image_from_image2d(temp_buf_float, 4);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    __global float* input_ptr = (__global float*)img1.ptr;\n\
    if(length > 0)\n\
    {\n\
        __global vxc_short8* output_ptr = (__global vxc_short8*)img2.ptr;\n\
        float4 src0 = vload4(0, input_ptr + gidx * 8);\n\
        float4 src1 = vload4(1, input_ptr + gidx * 8);\n\
        half4 data0, data1;\n\
        _viv_asm(CONV, data0, src0);\n\
        _viv_asm(CONV, data1, src1);\n\
        vxc_half8 tmp;\n\
        vxc_short8 dst;\n\
        VXC_DP2x8(tmp, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractHalf8_2x8);\n\
        _viv_asm(COPY, dst, tmp, 16);\n\
        output_ptr[gidx] = dst;\n\
    }\n\
    __global short* output_ptr1 = (__global short*)img2.ptr;\n\
    for(int i = gidx; i < res; i += get_global_size(0))\n\
    {\n\
        float src = input_ptr[length + i];\n\
        half data;\n\
        _viv_asm(CONV, data, src);\n\
        short dst;\n\
        _viv_asm(COPY, dst, data, 4);\n\
        output_ptr1[length + i] = dst;\n\
    }\n\
}\n\
\n\
__kernel void scatter_nd_update_reduction_conv_BF16(\n\
    __read_only image2d_t  temp_buf_float,\n\
    __read_only image2d_t  link_buf,\n\
    image2d_t  output,\n\
    int length, int res)\n\
{\n\
    int gidx = get_global_id(0);\n\
    Image img1 = create_image_from_image2d(temp_buf_float, 4);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    __global float* input_ptr = (__global float*)img1.ptr;\n\
    if(length > 0)\n\
    {\n\
        __global vxc_short8* output_ptr = (__global vxc_short8*)img2.ptr;\n\
        float4 src0 = vload4(0, input_ptr + gidx * 8);\n\
        float4 src1 = vload4(1, input_ptr + gidx * 8);\n\
        vxc_short8 dst0, dst1, dst;\n\
        _viv_asm(COPY, dst0, src0, 16);\n\
        _viv_asm(COPY, dst1, src1, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        output_ptr[gidx] = dst;\n\
    }\n\
    __global short* output_ptr1 = (__global short*)img2.ptr;\n\
    for(int i = gidx; i < res; i += get_global_size(0))\n\
    {\n\
        float src = input_ptr[length + i];\n\
        vxc_short8 data;\n\
        _viv_asm(COPY, data, src, 4);\n\
        output_ptr1[length + i] = data.x;\n\
    }\n\
}\n\
"; /* end of scatter_nd_update_reduction_conv_vx*/

static const char scatter_nd_update_special_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
_viv_uniform int offsetW;\n\
_viv_uniform int offset_idx;\n\
\n\
#define SCATTER_ND_UPDATE_REF2OUT_8BITS(src0_type, data_type) \\\n\
__kernel void scatter_nd_update_ref2out_##src0_type##to##src0_type( \\\n\
    __read_only image2d_t   input_ref, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  output0 \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img0 = create_image_from_image2d(input_ref, 1); \\\n\
    Image img1 = create_image_from_image2d(temp_ref, 1); \\\n\
    __global data_type* in_ptr = (__global data_type*)img0.ptr; \\\n\
    __global data_type* out_ptr = (__global data_type*)img1.ptr; \\\n\
    data_type src, dst; \\\n\
    src = in_ptr[gidx]; \\\n\
    vxc_ushort8 mp0; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(dst, src, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(dst, src, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Hi_2x8); \\\n\
    out_ptr[gidx] = dst; \\\n\
}\n\
SCATTER_ND_UPDATE_REF2OUT_8BITS(U8,  vxc_uchar16)\n\
SCATTER_ND_UPDATE_REF2OUT_8BITS(I8,  vxc_char16)\n\
\n\
#define SCATTER_ND_UPDATE_UPDATE2REF_8BITS(src0_type, data_type) \\\n\
__kernel void scatter_nd_update_update2ref_##src0_type##to##src0_type##_16x( \\\n\
    __read_only image2d_t   input_index, \\\n\
    __read_only image2d_t   input_update, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  input0, \\\n\
    image2d_t  output1, \\\n\
    int width, int area, int vol, int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input_index, 4); \\\n\
    Image img2 = create_image_from_image2d(input_update, 1); \\\n\
    Image img3 = create_image_from_image2d(temp_ref, 1); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global data_type* update_ptr = (__global data_type*)img2.ptr; \\\n\
    __global data_type* output_ptr = (__global data_type*)img3.ptr; \\\n\
    data_type dst; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim + offset_idx); \\\n\
    data_type src = update_ptr[gidy * update_width + gidx]; \\\n\
    int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst, src, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    VXC_DP2x8(dst, src, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Hi_2x8); \\\n\
    output_ptr[loc] = dst; \\\n\
}\n\
SCATTER_ND_UPDATE_UPDATE2REF_8BITS(U8,  vxc_uchar16)\n\
SCATTER_ND_UPDATE_UPDATE2REF_8BITS(I8,  vxc_char16)\n\
\n\
#define SCATTER_ND_UPDATE_COPY2OUT(src0_type, data_type, element_size) \\\n\
__kernel void scatter_nd_update_cpy2out_##src0_type##to##src0_type( \\\n\
    __read_only image2d_t   temp_ref, \\\n\
    image2d_t  input1, \\\n\
    image2d_t  output \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img0 = create_image_from_image2d(temp_ref, element_size); \\\n\
    Image img1 = create_image_from_image2d(output, element_size); \\\n\
    __global data_type* in_ptr = (__global data_type*)img0.ptr; \\\n\
    __global data_type* out_ptr = (__global data_type*)img1.ptr; \\\n\
    out_ptr[gidx] = in_ptr[gidx]; \\\n\
}\n\
SCATTER_ND_UPDATE_COPY2OUT(U8,  vxc_uchar16, 1)\n\
SCATTER_ND_UPDATE_COPY2OUT(I8,  vxc_char16, 1)\n\
SCATTER_ND_UPDATE_COPY2OUT(U16,  vxc_ushort8, 2)\n\
SCATTER_ND_UPDATE_COPY2OUT(I16,  vxc_short8, 2)\n\
\n\
#define SCATTER_ND_UPDATE_REF2OUT_16BITS(src0_type, data_type) \\\n\
__kernel void scatter_nd_update_ref2out_##src0_type##to##src0_type( \\\n\
    __read_only image2d_t   input_ref, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  output0 \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img0 = create_image_from_image2d(input_ref, 2); \\\n\
    Image img1 = create_image_from_image2d(temp_ref, 2); \\\n\
    __global data_type* in_ptr = (__global data_type*)img0.ptr; \\\n\
    __global data_type* out_ptr = (__global data_type*)img1.ptr; \\\n\
    data_type src, dst; \\\n\
    src = in_ptr[gidx]; \\\n\
    vxc_ushort8 mp0; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(dst, src, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift0_Lo_2x8); \\\n\
    out_ptr[gidx] = dst; \\\n\
}\n\
SCATTER_ND_UPDATE_REF2OUT_16BITS(U16,  vxc_ushort8)\n\
SCATTER_ND_UPDATE_REF2OUT_16BITS(I16,  vxc_short8)\n\
\n\
#define SCATTER_ND_UPDATE_UPDATE2REF_16BITS(src0_type, data_type) \\\n\
__kernel void scatter_nd_update_update2ref_##src0_type##to##src0_type##_16x( \\\n\
    __read_only image2d_t   input_index, \\\n\
    __read_only image2d_t   input_update, \\\n\
    image2d_t  temp_ref, \\\n\
    image2d_t  input0, \\\n\
    image2d_t  output1, \\\n\
    int width, int area, int vol, int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input_index, 4); \\\n\
    Image img2 = create_image_from_image2d(input_update, 2); \\\n\
    Image img3 = create_image_from_image2d(temp_ref, 2); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global data_type* update_ptr = (__global data_type*)img2.ptr; \\\n\
    __global data_type* output_ptr = (__global data_type*)img3.ptr; \\\n\
    data_type dst; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim + offset_idx); \\\n\
    data_type src = update_ptr[gidy * update_width + gidx]; \\\n\
    int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    VXC_DP2x8(dst, src, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift1_Lo_2x8); \\\n\
    output_ptr[loc] = dst; \\\n\
}\n\
SCATTER_ND_UPDATE_UPDATE2REF_16BITS(U16,  vxc_ushort8)\n\
SCATTER_ND_UPDATE_UPDATE2REF_16BITS(I16,  vxc_short8)\n\
\n\
__kernel void scatter_nd_update_ref2out_F16toF16(\n\
    __read_only image2d_t   input_ref,\n\
    image2d_t  temp_ref,\n\
    image2d_t  output0\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    Image img0 = create_image_from_image2d(input_ref, 2);\n\
    Image img1 = create_image_from_image2d(temp_ref, 2);\n\
    __global vxc_ushort8* in_ptr = (__global vxc_ushort8*)img0.ptr;\n\
    __global vxc_ushort8* out_ptr = (__global vxc_ushort8*)img1.ptr;\n\
    out_ptr[gidx] = in_ptr[gidx];\n\
}\n\
\n\
__kernel void scatter_nd_update_update2ref_F16toF16_16x(\n\
    __read_only image2d_t   input_index,\n\
    __read_only image2d_t   input_update,\n\
    image2d_t  temp_ref,\n\
    image2d_t  input0,\n\
    image2d_t  output1,\n\
    int width, int area, int vol, int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    Image img1 = create_image_from_image2d(input_index, 4);\n\
    Image img2 = create_image_from_image2d(input_update, 2);\n\
    Image img3 = create_image_from_image2d(temp_ref, 2);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global vxc_ushort8* update_ptr = (__global vxc_ushort8*)img2.ptr;\n\
    __global vxc_ushort8* output_ptr = (__global vxc_ushort8*)img3.ptr;\n\
\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim + offset_idx);\n\
    int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
    int loc = idx * output_width + gidx;\n\
    output_ptr[loc] = update_ptr[gidy * update_width + gidx];\n\
}\n\
"; /* end of scatter_nd_update_special_vx*/

static const char select_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvConditiontoDst_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define SELECT_INT(type_name, read_fun, write_fun) \\\n\
    type_name src0, src1, dst, value; \\\n\
    vxc_char8 value_tmp; \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    read_fun(src0, input0, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, input1, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
             uniU8MulAndPostShift0_Lo_2x8); \\\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
             uniU8MulAndPostShift1_Lo_2x8); \\\n\
    read_fun(value_tmp, condition, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(value, value_tmp, value_tmp,\\\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvConditiontoDst_2x8); \\\n\
    dst = (value != 0 ? src0 : src1); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define SELECT_INT_FUN(cond_name, src_name, dst_name, type_name) \\\n\
__kernel void select_##cond_name##_##src_name##_##src_name##to##dst_name( \\\n\
    __read_only  image2d_array_t   condition, \\\n\
    __read_only  image2d_array_t   input0, \\\n\
    __read_only  image2d_array_t   input1, \\\n\
    __write_only image2d_array_t   output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    SELECT_INT(type_name, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
SELECT_INT_FUN(I8, I8,  I8,  vxc_char8)\n\
SELECT_INT_FUN(I8, U8,  U8,  vxc_uchar8)\n\
SELECT_INT_FUN(I8, I16, I16, vxc_short8)\n\
\n\
#define SELECT_INT_FUN_2D(cond_name, src_name, dst_name, type_name) \\\n\
__kernel void select_##cond_name##_##src_name##_##src_name##to##dst_name##_2D( \\\n\
    __read_only  image2d_array_t   condition, \\\n\
    __read_only  image2d_array_t   input0, \\\n\
    __read_only  image2d_array_t   input1, \\\n\
    __write_only image2d_array_t   output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    SELECT_INT(type_name, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
SELECT_INT_FUN_2D(I8, I8,  I8,  vxc_char8)\n\
SELECT_INT_FUN_2D(I8, U8,  U8,  vxc_uchar8)\n\
SELECT_INT_FUN_2D(I8, I16, I16, vxc_short8)\n\
\n\
#define SELECT_HALF(read_fun, write_fun) \\\n\
    vxc_short8 src0, src1, dst, value; \\\n\
    vxc_char8 value_tmp; \\\n\
    read_fun(src0, input0, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, input1, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(value_tmp, condition, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(value, value_tmp, value_tmp,\\\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvConditiontoDst_2x8); \\\n\
    dst = (value != 0 ? src0 : src1); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void select_I8_F16_F16toF16(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SELECT_HALF(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void select_I8_F16_F16toF16_2D(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    SELECT_HALF(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define SELECT_HYBRID(src0_type, copy0_type, src1_type, copy1_type, dst_type, save_type, read_fun, write_fun) \\\n\
    save_type dst, value; \\\n\
    save_type dst0, dst1; \\\n\
    dst_type value0, value1; \\\n\
    src0_type src0; \\\n\
    src1_type src1; \\\n\
    copy0_type v0; \\\n\
    copy1_type v1; \\\n\
    vxc_char8 value_tmp; \\\n\
    vxc_ushort8 mp0, mp1; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    read_fun(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, v0, src0, 16); \\\n\
    read_fun(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, v1, src1, 16); \\\n\
    VXC_DP2x8(value0, v0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
             uniU8MulAndPostShift0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, value0, 16); \\\n\
    VXC_DP2x8(value1, v1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
             uniU8MulAndPostShift1_Lo_2x8); \\\n\
    _viv_asm(COPY, dst1, value1, 16); \\\n\
    read_fun(value_tmp, condition, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(value, value_tmp, value_tmp,\\\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvConditiontoDst_2x8); \\\n\
    dst = (value != 0 ? dst0 : dst1); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define SELECT_HYBRID_FUN(name, src0_type, copy0_type, src1_type, copy1_type, dst_type, save_type) \\\n\
__kernel void select_##name( \\\n\
    __read_only  image2d_array_t   condition, \\\n\
    __read_only  image2d_array_t   input0, \\\n\
    __read_only  image2d_array_t   input1, \\\n\
    __write_only image2d_array_t   output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    SELECT_HYBRID(src0_type, copy0_type, src1_type, copy1_type, dst_type, save_type,\\\n\
            VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
SELECT_HYBRID_FUN(I8_F16_U8toF16,  vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_U8_F16toF16,  vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_F16_I8toF16,  vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_I8_F16toF16,  vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_F16_I16toF16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_I16_F16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_F16_U8toU8,   vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_uchar8, vxc_uchar8)\n\
SELECT_HYBRID_FUN(I8_U8_F16toU8,   vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_uchar8, vxc_uchar8)\n\
SELECT_HYBRID_FUN(I8_F16_I8toI8,   vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_char8,  vxc_char8)\n\
SELECT_HYBRID_FUN(I8_I8_F16toI8,   vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_char8,  vxc_char8)\n\
SELECT_HYBRID_FUN(I8_F16_I16toI16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_short8, vxc_short8)\n\
SELECT_HYBRID_FUN(I8_I16_F16toI16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8, vxc_short8)\n\
SELECT_HYBRID_FUN(I8_U8_U8toF16,   vxc_uchar8,  vxc_uchar8,  vxc_uchar8,  vxc_uchar8,  vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_I8_I8toF16,   vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN(I8_I16_I16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,  vxc_short8)\n\
\n\
#define SELECT_HYBRID_FUN_2D(name, src0_type, copy0_type, src1_type, copy1_type, dst_type, save_type) \\\n\
__kernel void select_##name##_2D( \\\n\
    __read_only  image2d_array_t   condition, \\\n\
    __read_only  image2d_array_t   input0, \\\n\
    __read_only  image2d_array_t   input1, \\\n\
    __write_only image2d_array_t   output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    SELECT_HYBRID(src0_type, copy0_type, src1_type, copy1_type, dst_type, save_type, \\\n\
            VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
SELECT_HYBRID_FUN_2D(I8_F16_U8toF16,  vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_half8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_U8_F16toF16,  vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_half8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_F16_I8toF16,  vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_half8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_I8_F16toF16,  vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_half8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_F16_I16toF16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_half8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_I16_F16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_half8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_F16_U8toU8,   vxc_short8,  vxc_half8,   vxc_uchar16, vxc_uchar16, vxc_uchar8, vxc_uchar8)\n\
SELECT_HYBRID_FUN_2D(I8_U8_F16toU8,   vxc_uchar16, vxc_uchar16, vxc_short8,  vxc_half8,   vxc_uchar8, vxc_uchar8)\n\
SELECT_HYBRID_FUN_2D(I8_F16_I8toI8,   vxc_short8,  vxc_half8,   vxc_char16,  vxc_char16,  vxc_char8,  vxc_char8)\n\
SELECT_HYBRID_FUN_2D(I8_I8_F16toI8,   vxc_char16,  vxc_char16,  vxc_short8,  vxc_half8,   vxc_char8,  vxc_char8)\n\
SELECT_HYBRID_FUN_2D(I8_F16_I16toI16, vxc_short8,  vxc_half8,   vxc_short8,  vxc_short8,  vxc_short8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_I16_F16toI16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,   vxc_short8, vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_U8_U8toF16,   vxc_uchar8,  vxc_uchar8,  vxc_uchar8,  vxc_uchar8,  vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_I8_I8toF16,   vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   vxc_half8,  vxc_short8)\n\
SELECT_HYBRID_FUN_2D(I8_I16_I16toF16, vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  vxc_half8,  vxc_short8)\n\
\n\
#define SELECT_HALF_TO_QINT(read_fun, write_fun, dst_type) \\\n\
    vxc_short8 src0, src1, tmp_dst, value; \\\n\
    vxc_half8 data; \\\n\
    dst_type dst; \\\n\
    vxc_char8 value_tmp; \\\n\
    read_fun(src0, input0, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, input1, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(value_tmp, condition, coord, 0, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(value, value_tmp, value_tmp,\\\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvConditiontoDst_2x8); \\\n\
    tmp_dst = (value != 0 ? src0 : src1); \\\n\
    _viv_asm(COPY, data, tmp_dst, 16); \\\n\
    vxc_ushort8 mp0; \\\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(dst, data, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
            uniU8MulAndPostShift0_Lo_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void select_I8_F16_F16toU8(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SELECT_HALF_TO_QINT(VXC_ReadImage2DArray, VXC_WriteImage2DArray, vxc_uchar16)\n\
}\n\
\n\
__kernel void select_I8_F16_F16toU8_2D(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    SELECT_HALF_TO_QINT(VXC_ReadImage, VXC_WriteImage, vxc_uchar16)\n\
}\n\
"; /* end of select_vx*/

static const char sequence_mask_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int output_ZP;\n\
_viv_uniform float outputVal1;\n\
\n\
#define SEQUENCE_MASK_QINT_TO_QINT_2D(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void sequence_mask_##src0_type_name##to##src1_type_name##_2D( \\\n\
    image2d_t input, image2d_t output, int maxLen) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int2 coord = (int2)(gidx, get_global_id(1)); \\\n\
    read_type src0; \\\n\
    VXC_ReadImage(src0, input, coord.yy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3); \\\n\
    float4 tmpData; \\\n\
    short zp = inputZP; \\\n\
    VXC_DP4x4(tmpData, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                 uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    int index = convert_int_rtz(tmpData.s0 * input_scale); \\\n\
    int4 data; \\\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP; \\\n\
    write_type dst; \\\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I8,  I8,  vxc_char16,  vxc_char16)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I16, I16, vxc_short8,  vxc_short8)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I16, U8,  vxc_short8,  vxc_uchar16)\n\
\n\
#define SEQUENCE_MASK_QINT_TO_QINT(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void sequence_mask_##src0_type_name##to##src1_type_name( \\\n\
    image2d_t input, image2d_array_t output, int maxLen) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0); \\\n\
    read_type src0; \\\n\
    VXC_ReadImage(src0, input, coord.yz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3); \\\n\
    float4 tmpData; \\\n\
    short zp = inputZP; \\\n\
    VXC_DP4x4(tmpData, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                 uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    int index = convert_int_rtz(tmpData.s0 * input_scale); \\\n\
    int4 data; \\\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP; \\\n\
    write_type dst; \\\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SEQUENCE_MASK_QINT_TO_QINT(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT(I8,  I8,  vxc_char16,  vxc_char16)\n\
SEQUENCE_MASK_QINT_TO_QINT(I16, I16, vxc_short8,  vxc_short8)\n\
SEQUENCE_MASK_QINT_TO_QINT(I16, U8,  vxc_short8,  vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
\n\
__kernel void sequence_mask_F16toF16_2D(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rtz(tmpData.x);\n\
    float4 data;\n\
    data = outIdx < index? outputVal1 : convert_float(output_ZP);\n\
    vxc_short8 dst;\n\
    half4 tmpVal;\n\
    _viv_asm(CONV, tmpVal, data);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output, coord, dst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void sequence_mask_F16toF16(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rtz(tmpData.x);\n\
    float4 data;\n\
    data = outIdx < index? outputVal1 : convert_float(output_ZP);\n\
    vxc_short8 dst;\n\
    half4 tmpVal;\n\
    _viv_asm(CONV, tmpVal, data);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, coord, dst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void sequence_mask_F16toU8_2D(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rtz(tmpData.x);\n\
    int4 data;\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP;\n\
    vxc_uchar16 dst;\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void sequence_mask_F16toU8(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rtz(tmpData.x);\n\
    int4 data;\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP;\n\
    vxc_uchar16 dst;\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of sequence_mask_vx*/

static const char signal_frame_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define SIGNAL_FRAME_8BITS_SH_IMPL(type) \\\n\
__kernel void signal_frame_##type##to##type \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             frame_step \\\n\
    ) \\\n\
{ \\\n\
    int inner = get_global_id(0); \\\n\
    int length_k = get_global_id(1); \\\n\
    int frames_id = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(inner, length_k, frames_id, frames_id); \\\n\
    int2 coord_in = (int2)(inner, frames_id * frame_step + length_k); \\\n\
 \\\n\
    vxc_uchar16 src; \\\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SIGNAL_FRAME_8BITS_SH_IMPL(U8)\n\
SIGNAL_FRAME_8BITS_SH_IMPL(I8)\n\
\n\
#define SIGNAL_FRAME_16BITS_SH_IMPL(type) \\\n\
__kernel void signal_frame_##type##to##type \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             frame_step \\\n\
    ) \\\n\
{ \\\n\
    int inner = get_global_id(0); \\\n\
    int length_k = get_global_id(1); \\\n\
    int frames_id = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(inner, length_k, frames_id, frames_id); \\\n\
    int2 coord_in = (int2)(inner, frames_id * frame_step + length_k); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SIGNAL_FRAME_16BITS_SH_IMPL(I16)\n\
SIGNAL_FRAME_16BITS_SH_IMPL(F16)\n\
SIGNAL_FRAME_16BITS_SH_IMPL(BF16)"; /* end of signal_frame_vx*/

static const char slice_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define SLICE_SAMLEFL_SH_IMPL(name, data_type, end_bin) \\\n\
__kernel void slice_##name##_I32to##name##_SAMEFL \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    data_type src; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_SAMLEFL_SH_IMPL(U8, vxc_uchar16, 15)\n\
SLICE_SAMLEFL_SH_IMPL(I16, vxc_short8, 7)\n\
\n\
\n\
#define SLICE_SAMLEFL_2D_SH_IMPL(name, data_type, end_bin) \\\n\
__kernel void slice_##name##_I32to##name##_SAMEFL_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    data_type src; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_SAMLEFL_2D_SH_IMPL(U8, vxc_uchar16, 15)\n\
SLICE_SAMLEFL_2D_SH_IMPL(I16, vxc_short8, 7)\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
#define SLICE_8BITSTO16BITS(name0, name1, src_type, dst_type, save_type) \\\n\
__kernel void slice_##name0##_I32to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    dst_type dst0; \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst0, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO16BITS(I8, F16, vxc_char16,  vxc_half8,  vxc_short8)\n\
SLICE_8BITSTO16BITS(U8, F16, vxc_uchar16, vxc_half8,  vxc_short8)\n\
\n\
#define SLICE_8BITSTO16BITS_2D(name0, name1, src_type, dst_type, save_type) \\\n\
__kernel void slice_##name0##_I32to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src; \\\n\
    dst_type dst0; \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst0, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO16BITS_2D(I8, F16, vxc_char16,  vxc_half8,  vxc_short8)\n\
SLICE_8BITSTO16BITS_2D(U8, F16, vxc_uchar16, vxc_half8,  vxc_short8)\n\
\n\
#define SLICE_8BITSTO8BITS(name0, name1, src_type, dst_type) \\\n\
__kernel void slice_##name0##_I32to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Hi_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO8BITS(I8, I8, vxc_char16,  vxc_char16)\n\
SLICE_8BITSTO8BITS(U8, U8, vxc_uchar16, vxc_uchar16)\n\
\n\
#define SLICE_8BITSTO8BITS_2D(name0, name1, src_type, dst_type) \\\n\
__kernel void slice_##name0##_I32to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Hi_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO8BITS_2D(I8, I8, vxc_char16,  vxc_char16)\n\
SLICE_8BITSTO8BITS_2D(U8, U8, vxc_uchar16, vxc_uchar16)\n\
\n\
#define SLICE_16BITS_TO(name0, name1, src_type, copy_type, dst_type, save_type) \\\n\
__kernel void slice_##name0##_I32to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    copy_type src0; \\\n\
    dst_type result; \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage2DArray(src0, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, src0, 16); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(result, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, result, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_16BITS_TO(F16, I8,  vxc_half8,   vxc_short8, vxc_char16,  vxc_char16)\n\
SLICE_16BITS_TO(F16, U8,  vxc_half8,   vxc_short8, vxc_uchar16, vxc_uchar16)\n\
SLICE_16BITS_TO(F16, I16, vxc_half8,   vxc_short8, vxc_short8,  vxc_short8)\n\
SLICE_16BITS_TO(I16, I16, vxc_short8,  vxc_short8, vxc_short8,  vxc_short8)\n\
SLICE_16BITS_TO(I16, F16, vxc_short8,  vxc_short8, vxc_half8,   vxc_short8)\n\
\n\
#define SLICE_16BITS_TO_2D(name0, name1, src_type, copy_type, dst_type, save_type) \\\n\
__kernel void slice_##name0##_I32to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src; \\\n\
    copy_type src0; \\\n\
    dst_type result; \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage(src0, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, src0, 16); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(result, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, result, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_16BITS_TO_2D(F16, I8,  vxc_half8,   vxc_short8, vxc_char16,  vxc_char16)\n\
SLICE_16BITS_TO_2D(F16, U8,  vxc_half8,   vxc_short8, vxc_uchar16, vxc_uchar16)\n\
SLICE_16BITS_TO_2D(F16, I16, vxc_half8,   vxc_short8, vxc_short8,  vxc_short8)\n\
SLICE_16BITS_TO_2D(I16, I16, vxc_short8,  vxc_short8, vxc_short8,  vxc_short8)\n\
SLICE_16BITS_TO_2D(I16, F16, vxc_short8,  vxc_short8, vxc_half8,   vxc_short8)\n\
"; /* end of slice_vx*/

static const char space2depth_internal_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtractEvenUint8Stride2_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddUint8Stride2_2x8;\n\
_viv_uniform VXC_512Bits uniExtractEvenFp16Stride2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractOddFp16Stride2_4x4;\n\
\n\
_viv_uniform int input_depth;\n\
\n\
#define SPACE2DEPTH_INTERNAL_QINT_TO_QINT(src0_type_name, src1_type_name, read_type) \\\n\
__kernel void space2depth_internal_##src0_type_name##to##src1_type_name( \\\n\
    image2d_array_t input, \\\n\
    image2d_array_t output, \\\n\
    int block_size_x, \\\n\
    int block_size_y \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    ushort stride_x = (ushort)block_size_x; \\\n\
    ushort stride_y = (ushort)block_size_y; \\\n\
    ushort sidx = (ushort)gidx; \\\n\
    ushort sidy = (ushort)gidy; \\\n\
    ushort tmpX = sidx % stride_x; \\\n\
    ushort tmpY = sidy % stride_y; \\\n\
    int tmpId0 = tmpX; \\\n\
    int tmpId1 = tmpY; \\\n\
    int4 coord_out = (int4)((int)(sidx / stride_x), (int)(sidy / stride_y), 0, 0); \\\n\
    coord_out.z = tmpId0 * input_depth + tmpId1 * block_size_x * input_depth + gidz; \\\n\
    VXC_WriteImage2DArray(output, coord_out, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT(U8, U8, vxc_uchar16)\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT(I8, I8, vxc_char16)\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT(I16, I16, vxc_short8)\n\
\n\
__kernel void space2depth_internal_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int block_size_x,\n\
    int block_size_y\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    vxc_short8 data, imgVal0;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    ushort stride_x = (ushort)block_size_x;\n\
    ushort stride_y = (ushort)block_size_y;\n\
    ushort sidx = (ushort)gidx;\n\
    ushort sidy = (ushort)gidy;\n\
    ushort tmpX = sidx % stride_x;\n\
    ushort tmpY = sidy % stride_y;\n\
    int tmpId0 = tmpX;\n\
    int tmpId1 = tmpY;\n\
    int4 coord_out = (int4)((int)(sidx / stride_x), (int)(sidy / stride_y), 0, 0);\n\
    coord_out.z = tmpId0 * input_depth + tmpId1 * block_size_x * input_depth + gidz;\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, data, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define SPACE2DEPTH_INTERNAL_QINT_TO_QINT_X2Y1(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void space2depth_internal_##src0_type_name##to##src1_type_name##_X2Y1( \\\n\
    image2d_array_t input, \\\n\
    image2d_array_t output, \\\n\
    int block_size_x, \\\n\
    int block_size_y \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_out = (int4)(gidx >> 1, gidy, gidz, 0); \\\n\
    int out_d1; \\\n\
    read_type imageData; \\\n\
    write_type  imgVal0, imgVal1; \\\n\
 \\\n\
    VXC_ReadImage2DArray(imageData, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    out_d1 = gidz + input_depth; \\\n\
 \\\n\
    VXC_DP2x8(imgVal0, imageData, imageData,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractEvenUint8Stride2_2x8); \\\n\
    VXC_DP2x8(imgVal1, imageData, imageData,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddUint8Stride2_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_out.z = out_d1; \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT_X2Y1(U8, U8, vxc_uchar16, vxc_uchar16)\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT_X2Y1(I8, I8, vxc_char16, vxc_char16)\n\
\n\
#define SPACE2DEPTH_INTERNAL_16BITS_X2Y1(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void space2depth_internal_##src0_type_name##to##src1_type_name##_X2Y1( \\\n\
    image2d_array_t input, \\\n\
    image2d_array_t output, \\\n\
    int block_size_x, \\\n\
    int block_size_y \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_out = (int4)(gidx >> 1, gidy, gidz, 0); \\\n\
    int out_d1; \\\n\
    read_type imageData; \\\n\
    write_type  imgVal0, imgVal1; \\\n\
 \\\n\
    VXC_ReadImage2DArray(imageData, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    out_d1 = gidz + input_depth; \\\n\
    VXC_DP4x4(imgVal0, imageData, imageData, \\\n\
                 VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractEvenFp16Stride2_4x4); \\\n\
    VXC_DP4x4(imgVal1, imageData, imageData, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractOddFp16Stride2_4x4); \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_out.z = out_d1; \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SPACE2DEPTH_INTERNAL_16BITS_X2Y1(I16, I16, vxc_short8, vxc_short8)\n\
SPACE2DEPTH_INTERNAL_16BITS_X2Y1(F16, F16, vxc_short8, vxc_short8)"; /* end of space2depth_internal_vx*/

static const char swish_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
\n\
float4 sigmoid_(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define SWISH_PROCESS(read_fun, write_fun, src_type, src_copy_type, convert_type, dst_type, dst_copy_type, \\\n\
                     INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
    float4 vecA, vecB, vecC, vecD; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * INSCALE + INTAIL; \\\n\
    vecB = vecB * INSCALE + INTAIL; \\\n\
    vecC = beta * vecA; \\\n\
    vecD = beta * vecB; \\\n\
    vecC = sigmoid_(vecC); \\\n\
    vecD = sigmoid_(vecD); \\\n\
    vecA = vecA * vecC; \\\n\
    vecB = vecB * vecD; \\\n\
    vecA = vecA * OUTSCALE + OUTZP; \\\n\
    vecB = vecB * OUTSCALE + OUTZP; \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define SWISH_FUNC(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, dst_copy_type,\\\n\
                   INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void swish_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    SWISH_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, src_type, src_copy_type, convert_type, \\\n\
                  dst_type, dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
SWISH_FUNC(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
SWISH_FUNC(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
SWISH_FUNC(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
SWISH_FUNC(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
SWISH_FUNC(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
SWISH_FUNC(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
SWISH_FUNC(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, \\\n\
inputScale, inputTail, outputScale, outputZP)\n\
SWISH_FUNC(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
SWISH_FUNC(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
SWISH_FUNC(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
#define SWISH_FUNC_2D(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, \\\n\
                     dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void swish_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    SWISH_PROCESS(VXC_ReadImage, VXC_WriteImage, src_type, src_copy_type, convert_type, dst_type, \\\n\
                  dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
SWISH_FUNC_2D(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
SWISH_FUNC_2D(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
SWISH_FUNC_2D(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
SWISH_FUNC_2D(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
SWISH_FUNC_2D(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
SWISH_FUNC_2D(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
SWISH_FUNC_2D(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, \\\n\
inputScale, inputTail, outputScale, outputZP)\n\
SWISH_FUNC_2D(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
SWISH_FUNC_2D(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
SWISH_FUNC_2D(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define SWISH_BF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    float4 vecA, vecB, vecC, vecD; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecC = beta * vecA; \\\n\
    vecD = beta * vecB; \\\n\
    vecC = sigmoid_(vecC); \\\n\
    vecD = sigmoid_(vecD); \\\n\
    vecA = vecA * vecC; \\\n\
    vecB = vecB * vecD; \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void swish_BF16toBF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_BF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray);\n\
}\n\
\n\
__kernel void swish_BF16toBF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_BF16_PROCESS(VXC_ReadImage, VXC_WriteImage);\n\
}\n\
"; /* end of swish_vx*/

static const char tensorstackconcat_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void tensorstackconcat_16bits\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.w = 0;\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void tensorstackconcat_8bits\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int idx = coord.x;\n\
    vxc_char16 src0, src1;\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void tensorstackconcat_16bits_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage(output, coord.xy, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void tensorstackconcat_8bits_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int idx = coord.x;\n\
    vxc_char16 src0, src1;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage(output, coord.xy, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of tensorstackconcat_vx*/

static const char tile_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int lastWorkItem;\n\
\n\
#define TILE_3D(name0, name1, name2, remainder, type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out; \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
 \\\n\
    int batch_id = (short)coord.z / (short)depthIn; \\\n\
    coord.z = (short)coord.z % (short)depthIn; \\\n\
    coord_out = coord; \\\n\
 \\\n\
    for (int w = 0; w < multiples_3; w++) \\\n\
    { \\\n\
        int batch = batchIn * w + batch_id; \\\n\
 \\\n\
        for(int z = 0; z < multiples_2; z++) \\\n\
        { \\\n\
            coord_out.z = coord.z + z * depthIn + batch * depthOut; \\\n\
 \\\n\
            for (int y = 0; y < multiples_1; y++) \\\n\
            { \\\n\
                coord_out.y = coord.y + y * height; \\\n\
 \\\n\
                for (int x = 0; x < multiples_0; x++) \\\n\
                { \\\n\
                    coord_out.x = coord.x + x * width; \\\n\
                    if (isLastItem) \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, src, \\\n\
                            VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                    else \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, src, \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
}\n\
TILE_3D(U8, U8, 1, 0, vxc_uchar8)\n\
TILE_3D(U8, U8, 2, 1, vxc_uchar8)\n\
TILE_3D(U8, U8, 3, 2, vxc_uchar8)\n\
TILE_3D(U8, U8, 4, 3, vxc_uchar8)\n\
TILE_3D(U8, U8, 5, 4, vxc_uchar8)\n\
TILE_3D(U8, U8, 6, 5, vxc_uchar8)\n\
TILE_3D(U8, U8, 7, 6, vxc_uchar8)\n\
TILE_3D(U8, U8, 0, 7, vxc_uchar8)\n\
\n\
TILE_3D(I16, I16, 1, 0, vxc_short8)\n\
TILE_3D(I16, I16, 2, 1, vxc_short8)\n\
TILE_3D(I16, I16, 3, 2, vxc_short8)\n\
TILE_3D(I16, I16, 4, 3, vxc_short8)\n\
TILE_3D(I16, I16, 5, 4, vxc_short8)\n\
TILE_3D(I16, I16, 6, 5, vxc_short8)\n\
TILE_3D(I16, I16, 7, 6, vxc_short8)\n\
TILE_3D(I16, I16, 0, 7, vxc_short8)\n\
\n\
\n\
#define TILE_2D(name0, name1, name2, remainder, type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1##_2D( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
    type src; \\\n\
    VXC_ReadImage(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            if (isLastItem) \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            else \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            coord.x += width; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = get_global_id(0); \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D(U8, U8, 1, 0, vxc_uchar8)\n\
TILE_2D(U8, U8, 2, 1, vxc_uchar8)\n\
TILE_2D(U8, U8, 3, 2, vxc_uchar8)\n\
TILE_2D(U8, U8, 4, 3, vxc_uchar8)\n\
TILE_2D(U8, U8, 5, 4, vxc_uchar8)\n\
TILE_2D(U8, U8, 6, 5, vxc_uchar8)\n\
TILE_2D(U8, U8, 7, 6, vxc_uchar8)\n\
TILE_2D(U8, U8, 0, 7, vxc_uchar8)\n\
\n\
TILE_2D(I16, I16, 1, 0, vxc_short8)\n\
TILE_2D(I16, I16, 2, 1, vxc_short8)\n\
TILE_2D(I16, I16, 3, 2, vxc_short8)\n\
TILE_2D(I16, I16, 4, 3, vxc_short8)\n\
TILE_2D(I16, I16, 5, 4, vxc_short8)\n\
TILE_2D(I16, I16, 6, 5, vxc_short8)\n\
TILE_2D(I16, I16, 7, 6, vxc_short8)\n\
TILE_2D(I16, I16, 0, 7, vxc_short8)\n\
\n\
#define TILE_2D_1TON(name0, name1, type) \\\n\
__kernel void tile_1toN_##name0##to##name1##_2D( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
    type src; \\\n\
    VXC_ReadImage(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.x += 8; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = 0; \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D_1TON(U8,  U8, vxc_uchar8)\n\
TILE_2D_1TON(I16, I16, vxc_short8)\n\
"; /* end of tile_vx*/

static const char tile_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Lo_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform int lastWorkItem;\n\
\n\
#define TILE_3D_MIX(name0, name1, name2, remainder, type, out_type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out; \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    type src; \\\n\
    vxc_half8  src1; \\\n\
    out_type dst; \\\n\
    VXC_ReadImage2DArray(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
 \\\n\
    int batch_id = (short)coord.z / (short)depthIn; \\\n\
    coord.z = (short)coord.z % (short)depthIn; \\\n\
    coord_out = coord; \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(src1, src, multiplier,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    _viv_asm(COPY, dst, src1, 16); \\\n\
 \\\n\
    for (int w = 0; w < multiples_3; w++) \\\n\
    { \\\n\
        int batch = batchIn * w + batch_id; \\\n\
 \\\n\
        for(int z = 0; z < multiples_2; z++) \\\n\
        { \\\n\
            coord_out.z = coord.z + z * depthIn + batch * depthOut; \\\n\
 \\\n\
            for (int y = 0; y < multiples_1; y++) \\\n\
            { \\\n\
                coord_out.y = coord.y + y * height; \\\n\
 \\\n\
                for (int x = 0; x < multiples_0; x++) \\\n\
                { \\\n\
                    coord_out.x = coord.x + x * width; \\\n\
                    if (isLastItem) \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, dst, \\\n\
                            VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                    else \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, dst, \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
}\n\
TILE_3D_MIX(U8, F16, 1, 0, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 2, 1, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 3, 2, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 4, 3, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 5, 4, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 6, 5, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 7, 6, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 0, 7, vxc_uchar8, vxc_short8)\n\
\n\
\n\
#define TILE_2D_MIX(name0, name1, name2, remainder, type, out_type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1##_2D( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
    type src; \\\n\
    vxc_half8  src1; \\\n\
    out_type dst; \\\n\
    VXC_ReadImage(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(src1, src, multiplier,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    _viv_asm(COPY, dst, src1, 16); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            if (isLastItem) \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            else \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            coord.x += width; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = get_global_id(0); \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D_MIX(U8, F16, 1, 0, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 2, 1, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 3, 2, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 4, 3, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 5, 4, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 6, 5, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 7, 6, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 0, 7, vxc_uchar8, vxc_short8)\n\
"; /* end of tile_mix_vx*/

static const char tiny_yolov4_postprocess_box_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE    (1.44269502f)\n\
\n\
float4 sigmoid4(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
\n\
float4 exp4(float4 x)\n\
{\n\
    x *= logE;\n\
    return exp2(x);\n\
}\n\
\n\
#define CONST0      (1.0499999523162842f)\n\
#define CONST1      (0.0250000003725290f)\n\
\n\
_viv_uniform VXC_512Bits uniDatatoFloat32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFloat32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDataTranspose_0_2x8;\n\
_viv_uniform VXC_512Bits uniDataTranspose_1_2x8;\n\
_viv_uniform float input0_scale;\n\
_viv_uniform float input0_tail;\n\
_viv_uniform float input1_scale;\n\
_viv_uniform float input1_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float CONST2;\n\
__kernel void tiny_yolov4_postprocess_box_U8_U8toU8\n\
    (\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            bias_0,\n\
                 float            bias_1\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(0));\n\
\n\
    vxc_uchar16 src0, src1, src2, src3;\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input0, coord.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input0, coord.xy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(src2, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src3, input1, coord.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.zw += (int2)(2, 3);\n\
\n\
    float4 data0, data1, data2, data3, data;\n\
    VXC_DP4x4(data0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFloat32_0_4x4);\n\
    data0 = data0 * input0_scale + input0_tail;\n\
    data0 = sigmoid4(data0);\n\
    data0 = data0 * CONST0 - CONST1;\n\
\n\
    VXC_DP4x4(data, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFloat32_0_4x4);\n\
    data = data * input1_scale + input1_tail;\n\
    data0 = data0 * CONST2 + data * CONST2;\n\
\n\
    VXC_DP4x4(data1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFloat32_1_4x4);\n\
    data1 = data1 * input0_scale + input0_tail;\n\
    data1 = sigmoid4(data1);\n\
    data1 = data1 * CONST0 - CONST1;\n\
\n\
    VXC_DP4x4(data, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFloat32_0_4x4);\n\
    data = data * input1_scale + input1_tail;\n\
    data1 = data1 * CONST2 + data * CONST2;\n\
\n\
    VXC_DP4x4(data2, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFloat32_0_4x4);\n\
    data2 = data2 * input0_scale + input0_tail;\n\
    data2 = exp4(data2) * bias_0;\n\
\n\
    VXC_DP4x4(data3, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFloat32_1_4x4);\n\
    data3 = data3 * input0_scale + input0_tail;\n\
    data3 = exp4(data3) * bias_1;\n\
\n\
    data0 = data0 * output_scale + output_zp;\n\
    data1 = data1 * output_scale + output_zp;\n\
\n\
    int4 dst0 = convert_int4_rte(data0);\n\
    int4 dst1 = convert_int4_rte(data1);\n\
    VXC_DP2x8(src1, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
    data2 = data2 * output_scale + output_zp;\n\
    data3 = data3 * output_scale + output_zp;\n\
    dst0 = convert_int4_rte(data2);\n\
    dst1 = convert_int4_rte(data3);\n\
    VXC_DP2x8(src1, dst0, dst1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8);\n\
\n\
    VXC_DP2x8(src0, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniDataTranspose_0_2x8);\n\
    VXC_DP2x8(src0, src1, src1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniDataTranspose_1_2x8);\n\
\n\
    VXC_WriteImage(output, coord.yx, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, src0, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.yz, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.yw, src0, VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of tiny_yolov4_postprocess_box_vx*/

static const char tiny_yolov4_postprocess_confidence_vx[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8TimesU8_0_4x4;\n\
_viv_uniform VXC_512Bits uniU8PlusU8_trans_0_2x8;\n\
_viv_uniform VXC_512Bits uniU8PlusU8_trans_1_2x8;\n\
_viv_uniform VXC_512Bits uniU16TimesMultiplier_PostShift_2x8;\n\
_viv_uniform int output_zp;\n\
\n\
__kernel void tiny_yolov4_postprocess_conf_U8toU8\n\
(\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, get_global_id(0));\n\
\n\
    vxc_uchar16 src0, src1, src2, src3, src4;\n\
\n\
    VXC_ReadImage(src0, input, coord.wz, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 data0, data1;\n\
\n\
    VXC_ReadImage(src1, input, coord.wy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input, coord.wy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src3, input, coord.wy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src4, input, coord.wy, VXC_5BITOFFSET_XY(0, 4), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.zw = coord.xx + (int2)(2, 3);\n\
\n\
    VXC_DP4x4(data0, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8TimesU8_0_4x4);\n\
    VXC_DP4x4(data0, src0, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniU8TimesU8_0_4x4);\n\
    VXC_DP4x4(data1, src0, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8TimesU8_0_4x4);\n\
    VXC_DP4x4(data1, src0, src4, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0), uniU8TimesU8_0_4x4);\n\
\n\
    VXC_DP2x8(src1, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniU16TimesMultiplier_PostShift_2x8);\n\
    VXC_DP2x8(src1, data1, data1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\n\
        uniU16TimesMultiplier_PostShift_2x8);\n\
\n\
    uchar zp;\n\
    _viv_asm(COPY, zp, output_zp, 2);\n\
\n\
    VXC_DP2x8(src0, src1, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniU8PlusU8_trans_0_2x8);\n\
    VXC_DP2x8(src0, src1, zp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\n\
        uniU8PlusU8_trans_1_2x8);\n\
\n\
    VXC_WriteImage(output, coord.yx, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, src0, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.yz, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.yw, src0, VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of tiny_yolov4_postprocess_confidence_vx*/

static const char upsample_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniF16MulMultipiler_PostShft_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
#define UPSAMPLE_F16_U8TO_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0; \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_half8 src; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 axisData1; \\\n\
    vxc_uchar16 dout; \\\n\
    read_fun(din0, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 tmp; \\\n\
    uchar zp = 0; \\\n\
    _viv_asm(COPY, src, din0, 16); \\\n\
    VXC_DP2x8(tmp, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniF16MulMultipiler_PostShft_2x8); \\\n\
    vxc_uchar16 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(din, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    constAxis    = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    dinExpand    = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_F16_U8to_U8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_U8TO_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_U8to_U8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_U8TO_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits shortMulShort_8x8;\n\
_viv_uniform VXC_512Bits uniConvertFstFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform int upOutput_ZP;\n\
_viv_uniform float upOutput_Scale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
\n\
#define UPSAMPLE_F16_I16TO_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_short4 axisIn; \\\n\
    vxc_short8 dinExp, axisInExp, constAxis,axisData,tmpout; \\\n\
    vxc_half8 dout; \\\n\
    vxc_float4 tmpVal1, tmpVal2, convZp; \\\n\
    vxc_int4 tmpData1, tmpData2, tmpData3; \\\n\
    vxc_uchar8 result; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_short8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_short8)(1); \\\n\
    VXC_DP2x8(tmpout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), shortMulShort_8x8); \\\n\
    _viv_asm(COPY, dout, tmpout, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 /= upOutput_Scale; \\\n\
    tmpVal2 /= upOutput_Scale; \\\n\
    tmpData3 = isnotequal(tmpVal1, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal1 += convZp; \\\n\
    tmpData3 = isnotequal(tmpVal2, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal2 += convZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_short8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_short8)(1); \\\n\
    VXC_DP2x8(tmpout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), shortMulShort_8x8); \\\n\
    _viv_asm(COPY, dout, tmpout, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 /= upOutput_Scale; \\\n\
    tmpVal2 /= upOutput_Scale; \\\n\
    tmpData3 = isnotequal(tmpVal1, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal1 += convZp; \\\n\
    tmpData3 = isnotequal(tmpVal2, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal2 += convZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_F16_I16to_U8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_I16TO_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_I16to_U8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_I16TO_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform float scaleOut;\n\
_viv_uniform float outputZp;\n\
_viv_uniform VXC_512Bits ucharMulShort_8x8_2;\n\
\n\
#define UPSAMPLE_F16_U8TO_I8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_uchar4 axisIn; \\\n\
    vxc_short8 dinExp, tmpOut; \\\n\
    vxc_uchar8 axisInExp; \\\n\
    vxc_uchar8 constAxis; \\\n\
    vxc_uchar8 axisData; \\\n\
    vxc_half8 dout; \\\n\
    vxc_float4 tmpVal0, tmpVal1; \\\n\
    vxc_char8 result; \\\n\
    int4 tmpData1, tmpData2; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_uchar8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal0 = tmpVal0 * scaleOut + outputZp; \\\n\
    tmpVal1 = tmpVal1 * scaleOut + outputZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal0); \\\n\
    tmpData2 = convert_int4_rte(tmpVal1); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    coordOut.y += 1; \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal0 = tmpVal0 * scaleOut + outputZp; \\\n\
    tmpVal1 = tmpVal1 * scaleOut + outputZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal0); \\\n\
    tmpData2 = convert_int4_rte(tmpVal1); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_F16_U8to_I8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_U8TO_I8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_U8to_I8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_U8TO_I8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform float up_outFlScale_i16;\n\
\n\
#define UPSAMPLE_F16_U8TO_I16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_uchar4 axisIn; \\\n\
    vxc_short8 dinExp, tmpOut; \\\n\
    vxc_uchar8 axisInExp; \\\n\
    vxc_uchar8 constAxis; \\\n\
    vxc_uchar8 axisData; \\\n\
    half8 dout; \\\n\
    float4 tmpVal1, tmpVal2; \\\n\
    int4 tmpData1, tmpData2; \\\n\
    vxc_short8 result; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_uchar8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 *= up_outFlScale_i16; \\\n\
    tmpVal2 *= up_outFlScale_i16; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    coordOut.y += 1; \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 *= up_outFlScale_i16; \\\n\
    tmpVal2 *= up_outFlScale_i16; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_F16_U8to_I16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_U8TO_I16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_U8to_I16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_U8TO_I16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_F16_vx*/

static const char upsample_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
//--------------------------unpooling-------------------------\n\
_viv_uniform VXC_512Bits uniQuantInOutInt16_2x8;\n\
_viv_uniform VXC_512Bits ucharMulShort_2x8;\n\
\n\
#define UPSAMPLE_I16_U8TO_I16_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_uchar4 axisIn; \\\n\
    vxc_short8 dinExp; \\\n\
    vxc_uchar8 axisInExp; \\\n\
    vxc_uchar8 constAxis; \\\n\
    vxc_uchar8 axisData; \\\n\
    vxc_short8 axisData_short; \\\n\
    vxc_short8 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_uchar8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_I16_U8to_I16_SAME\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_U8TO_I16_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_U8to_I16_SAME_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_U8TO_I16_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define UPSAMPLE_I16_TO_I16_PROCESS(axis_type, axis_in_type, read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    axis_in_type axisIn; \\\n\
    vxc_short8 dinExp; \\\n\
    axis_type  axisInExp; \\\n\
    axis_type  constAxis; \\\n\
    axis_type  axisData; \\\n\
    vxc_short8 axisData_short; \\\n\
    vxc_short8 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (axis_type)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    VXC_DP2x8(dout, dout, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantInOutInt16_2x8); \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (axis_type)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    VXC_DP2x8(dout, dout, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantInOutInt16_2x8); \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_I16_U8to_I16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_U8to_I16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
__kernel void upsample_I16_I16to_I16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_I16to_I16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirInt16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertU8toI16_2x8;\n\
_viv_uniform float inScaleInt16;\n\
\n\
#define UPSAMPLE_I16_TO_F16_PROCESS(axis_type, axis_in_type, read_fun, write_fun) \\\n\
    vxc_short8 din; \\\n\
    axis_in_type axisIn; \\\n\
    vxc_short8 dinExp; \\\n\
    axis_type  axisInExp; \\\n\
    axis_type  constAxis; \\\n\
    axis_type  axisData; \\\n\
    vxc_short8 axisData_short; \\\n\
    vxc_short8 dout; \\\n\
    vxc_float4 tmpVal0, tmpVal1; \\\n\
    half4 tmpOut0; \\\n\
    vxc_short8 tmpOut1; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP4x4(tmpVal0, din, din, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertDirInt16Fp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inScaleInt16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    _viv_asm(COPY, tmpOut1, tmpOut0, 16); \\\n\
    dinExp    = tmpOut1.s00224466; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (axis_type)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(axisData_short, axisData, axisData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertU8toI16_2x8); \\\n\
    dout = axisData_short == 1 ? dinExp : 0; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (axis_type)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(axisData_short, axisData, axisData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertU8toI16_2x8); \\\n\
    dout = axisData_short == 1 ? dinExp : 0; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_I16_I16to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_I16to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
__kernel void upsample_I16_U8to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_U8to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_I16_vx*/

static const char upsample_I8_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
\n\
#define UPSAMPLE_I8_U8TO_I8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_char8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_char16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_char16 zpValue; \\\n\
    vxc_char16 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    zpValue = (char)input_ZP; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_I8_U8to_I8_SAME\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I8_U8TO_I8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I8_U8to_I8_SAME_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I8_U8TO_I8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform VXC_512Bits uniS16MoveValue_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
#define UPSAMPLE_I8_U8TO_I8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_char16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_char16 zpValue; \\\n\
    vxc_char16 dout; \\\n\
    vxc_char16 result, result_tmp; \\\n\
    zpValue = (char)input_ZP; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    vxc_short8 tmp; \\\n\
    short zp = input_ZP; \\\n\
    vxc_short8 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_I8_U8to_I8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I8_U8TO_I8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I8_U8to_I8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I8_U8TO_I8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertEndUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertTrdUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertFthUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8_2;\n\
_viv_uniform float scaleIn;\n\
_viv_uniform float inputTail;\n\
\n\
#define UPSAMPLE_I8_U8TO_F16_PROCESS(read_fun, write_fun) \\\n\
    vxc_char8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_char16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_char16 zpValue; \\\n\
    vxc_char16 dout; \\\n\
    zpValue = (char)input_ZP; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coordOut1.x += 8; \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_short8 rout0, rout1; \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertDirUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertEndUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertTrdUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFthUint8Fp32_4x4_2); \\\n\
    tmpVal0 = tmpVal0 * scaleIn + inputTail; \\\n\
    tmpVal1 = tmpVal1 * scaleIn + inputTail; \\\n\
    tmpVal2 = tmpVal2 * scaleIn + inputTail; \\\n\
    tmpVal3 = tmpVal3 * scaleIn + inputTail; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertDirUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertEndUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertTrdUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFthUint8Fp32_4x4_2); \\\n\
    tmpVal0 = tmpVal0 * scaleIn + inputTail; \\\n\
    tmpVal1 = tmpVal1 * scaleIn + inputTail; \\\n\
    tmpVal2 = tmpVal2 * scaleIn + inputTail; \\\n\
    tmpVal3 = tmpVal3 * scaleIn + inputTail; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    coordOut.y += 1; \\\n\
    coordOut1.y += 1; \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_I8_U8to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    int4 coordOut1 = coordOut;\n\
    UPSAMPLE_I8_U8TO_F16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I8_U8to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    int2 coordOut1 = coordOut;\n\
    UPSAMPLE_I8_U8TO_F16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_I8_vx*/

static const char upsample_U8_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
\n\
#define UPSAMPLE_U8_U8TO_U8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 zpValue; \\\n\
    vxc_uchar16 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    zpValue = (uchar)input_ZP; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_U8_U8to_U8_SAME\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_U8_U8TO_U8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_U8_U8to_U8_SAME_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_U8_U8TO_U8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform VXC_512Bits uniS16MoveValue_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
#define UPSAMPLE_U8_U8TO_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 zpValue; \\\n\
    vxc_uchar16 dout; \\\n\
    vxc_uchar16 result, result_tmp; \\\n\
    zpValue = (uchar)input_ZP; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    vxc_short8 tmp; \\\n\
    short zp = input_ZP; \\\n\
    vxc_short8 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_U8_U8to_U8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_U8_U8TO_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_U8_U8to_U8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_U8_U8TO_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniMulMinusZpUint8_4x4;\n\
_viv_uniform VXC_512Bits uniMulMinusZp2Uint8_4x4;\n\
_viv_uniform VXC_512Bits uniMulMinusZp3Uint8_4x4;\n\
_viv_uniform VXC_512Bits uniMulMinusZp4Uint8_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toInt16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertDirUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEndUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertTrdUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFthUint8Fp32_4x4;\n\
_viv_uniform float scaleU8Fp16;\n\
_viv_uniform int zpU8Fp16;\n\
\n\
#define UPSAMPLE_U8_U8TO_F16_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 axisData1; \\\n\
    vxc_uchar16 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coordOut1.x += 8; \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3, convZp; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_short8 rout0, rout1; \\\n\
    vxc_int4 tmpV0, tmpV1, tmpV2, tmpV3; \\\n\
    vxc_float4 tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
    short tmpZp = (short)(-zpU8Fp16); \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertDirUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertTrdUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertFthUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpV0, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZpUint8_4x4); \\\n\
    VXC_DP4x4(tmpV1, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp2Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV2, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp3Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV3, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp4Uint8_4x4); \\\n\
    _viv_asm(CONV, tmpData0, tmpV0); \\\n\
    _viv_asm(CONV, tmpData1, tmpV1); \\\n\
    _viv_asm(CONV, tmpData2, tmpV2); \\\n\
    _viv_asm(CONV, tmpData3, tmpV3); \\\n\
    tmpVal0 = (tmpVal0 + tmpData0) * scaleU8Fp16; \\\n\
    tmpVal1 = (tmpVal1 + tmpData1) * scaleU8Fp16; \\\n\
    tmpVal2 = (tmpVal2 + tmpData2) * scaleU8Fp16; \\\n\
    tmpVal3 = (tmpVal3 + tmpData3) * scaleU8Fp16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertDirUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertTrdUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertFthUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpV0, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZpUint8_4x4); \\\n\
    VXC_DP4x4(tmpV1, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp2Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV2, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp3Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV3, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp4Uint8_4x4); \\\n\
    _viv_asm(CONV, tmpData0, tmpV0); \\\n\
    _viv_asm(CONV, tmpData1, tmpV1); \\\n\
    _viv_asm(CONV, tmpData2, tmpV2); \\\n\
    _viv_asm(CONV, tmpData3, tmpV3); \\\n\
    tmpVal0 = (tmpVal0 + tmpData0) * scaleU8Fp16; \\\n\
    tmpVal1 = (tmpVal1 + tmpData1) * scaleU8Fp16; \\\n\
    tmpVal2 = (tmpVal2 + tmpData2) * scaleU8Fp16; \\\n\
    tmpVal3 = (tmpVal3 + tmpData3) * scaleU8Fp16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    coordOut.y += 1; \\\n\
    coordOut1.y += 1; \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_U8_U8to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord     = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut  = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    int4 coordOut1 = coordOut;\n\
    UPSAMPLE_U8_U8TO_F16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_U8_U8to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord     = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut  = (int2)(coord.x << 1, coord.y << 1);\n\
    int2 coordOut1 = coordOut;\n\
    UPSAMPLE_U8_U8TO_F16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_U8_vx*/

static const char upsamplescale_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertDatatoF32_4x4;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float tail;\n\
\n\
#define UPSAMPLE_SCALETO_FUN(src_name, dst_name, read_type, src_type, dst_type, write_type, conv_func) \\\n\
    __kernel void upsamplescale_##src_name##to##dst_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride, \\\n\
                 float            scale) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst_val; \\\n\
    write_type write_val; \\\n\
    VXC_ReadImage2DArray(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    coord.xy *= stride; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.w, baseAddr); \\\n\
    float4 data; \\\n\
    VXC_DP4x4(data, src_val, src_val, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertDatatoF32_4x4); \\\n\
    data = data * output_scale + tail; \\\n\
    _viv_asm(conv_func, dst_val, data); \\\n\
    _viv_asm(COPY, write_val, dst_val, 16); \\\n\
    int4 coord_out = coord; \\\n\
    for (int y = 0; y < stride; y++) \\\n\
    { \\\n\
        coord_out.x = coord.x; \\\n\
        for (int x = 0; x < stride; ) \\\n\
        { \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, write_val, \\\n\
                VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0)); \\\n\
            x++; \\\n\
            coord_out.x ++; \\\n\
        } \\\n\
        coord_out.y ++; \\\n\
    } \\\n\
}\n\
\n\
UPSAMPLE_SCALETO_FUN(F16, F16,  vxc_short8,  vxc_half8,   half4,  short4, CONV)\n\
UPSAMPLE_SCALETO_FUN(F16, I16,  vxc_short8,  vxc_half8,   int4,   short4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(F16, I8,   vxc_short8,  vxc_half8,   int4,   char4,  CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(F16, U8,   vxc_short8,  vxc_half8,   int4,   uchar4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(I16, I16,  vxc_short8,  vxc_short8,  int4,   short4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(I16, F16,  vxc_short8,  vxc_short8,  half4,  short4, CONV)\n\
UPSAMPLE_SCALETO_FUN(I8,  I8,   vxc_char16,  vxc_char16,  int4,   char4,  CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(I8,  F16,  vxc_short8,  vxc_short8,  half4,  short4, CONV)\n\
UPSAMPLE_SCALETO_FUN(U8,  U8,   vxc_uchar16, vxc_uchar16, int4,   uchar4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(U8,  F16,  vxc_short8,  vxc_short8,  half4,  short4, CONV)\n\
\n\
"; /* end of upsamplescale_vx*/

static const char upsamplescale_k2_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniUpScale2X_lo_2x8;\n\
_viv_uniform VXC_512Bits uniUpScale2X_hi_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define UPSAMPLE_SCALETO8B_FUN(src_name, dst_name, read_type, src_type, dst_type) \\\n\
    __kernel void upsamplescale_##src_name##to##dst_name##_K2( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride, \\\n\
                 float            scale) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst_val; \\\n\
    VXC_ReadImage2DArray(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    coord.xy <<= 1; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.w, baseAddr); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst_val, src_val, multiplier, \\\n\
          VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniUpScale2X_lo_2x8); \\\n\
    VXC_DP2x8(dst_val, src_val, multiplier, \\\n\
          VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniUpScale2X_hi_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst_val, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.y ++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst_val, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
UPSAMPLE_SCALETO8B_FUN(F16, I8,  vxc_short8,  vxc_half8,   vxc_char16)\n\
UPSAMPLE_SCALETO8B_FUN(F16, U8,  vxc_short8,  vxc_half8,   vxc_uchar16)\n\
UPSAMPLE_SCALETO8B_FUN(I8,  I8,  vxc_char16,  vxc_char16,  vxc_char16)\n\
UPSAMPLE_SCALETO8B_FUN(U8,  U8,  vxc_uchar16, vxc_uchar16, vxc_uchar16)\n\
\n\
#define UPSAMPLE_SCALETO16B_FUN(src_name, dst_name, read_type, src_type, dst_type, write_type) \\\n\
    __kernel void upsamplescale_##src_name##to##dst_name##_K2( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride, \\\n\
                 float            scale) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst0_val; \\\n\
    dst_type   dst1_val; \\\n\
    write_type write_val; \\\n\
    VXC_ReadImage2DArray(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    coord.xy <<= 1; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.w, baseAddr); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst0_val, src_val, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniUpScale2X_lo_2x8); \\\n\
    VXC_DP2x8(dst1_val, src_val, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniUpScale2X_hi_2x8); \\\n\
    _viv_asm(COPY, write_val, dst0_val, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.y ++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, write_val, dst1_val, 16); \\\n\
    coord.xy = coord.xy + (int2)(8, -1); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.y ++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
UPSAMPLE_SCALETO16B_FUN(F16, F16,  vxc_short8,  vxc_half8,   vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(F16, I16,  vxc_short8,  vxc_half8,   vxc_short8, vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(I8,  F16,  vxc_char16,  vxc_char16,  vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(U8,  F16,  vxc_uchar16, vxc_uchar16, vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(I16, F16,  vxc_short8,  vxc_short8,  vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(I16, I16,  vxc_short8,  vxc_short8,  vxc_short8, vxc_short8)\n\
"; /* end of upsamplescale_k2_vx*/

static const char vsi_nn_kernel_header_vx[] = "\n\
#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
typedef struct Image\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
} Image;\n\
\n\
inline uchar* get_image_ptr_from_coord(Image img, int2 coord)\n\
{\n\
    return img.ptr + coord.x * img.stride_x + coord.y * img.stride_y;\n\
}\n\
\n\
inline Image create_image_from_image2d(image2d_t input, int stride_x)\n\
{\n\
    int stride_y;\n\
#if (USE_40BITS_VA==0)\n\
    int8 desc;\n\
#else\n\
    int8 desc;\n\
    _viv_asm(GET_IMAGE_STRIDE, stride_y, input);\n\
#endif\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
    uint address = as_uint(desc.s0);\n\
\n\
#if (USE_40BITS_VA==0)\n\
    stride_y = desc.s1;\n\
#endif\n\
\n\
    Image img =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = stride_y\n\
    };\n\
\n\
    return img;\n\
}\n\
\n\
typedef struct Tensor\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
    int             stride_z;\n\
} Tensor;\n\
\n\
inline uchar* get_tensor_ptr_from_coord(Tensor t, int4 coord)\n\
{\n\
    return t.ptr + coord.x * t.stride_x + coord.y * t.stride_y + coord.z * t.stride_z;\n\
}\n\
\n\
inline Tensor create_tensor_from_image2d_array(image2d_array_t input, int stride_x)\n\
{\n\
    int8 desc;\n\
    int2 strides;\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
#if (USE_40BITS_VA==0)\n\
    strides.x = desc.s1;\n\
    strides.y = desc.s4;\n\
#else\n\
    _viv_asm(GET_IMAGE_STRIDE, strides, input);\n\
#endif\n\
    uint address = as_uint(desc.s0);\n\
\n\
    Tensor t =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = strides.x,\n\
        .stride_z                      = strides.y\n\
    };\n\
\n\
    return t;\n\
}\n\
\n\
#if (VX_VERSION==1)\n\
#define VXC_DP2x8_b_(dst, src0, src1, src2, info, uniform)\\\n\
do\\\n\
{\\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
} while (0)\n\
\n\
#define VXC_VertMin3_Integer(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    typeof (dst) tmp; \\\n\
    tmp  = min(src0, src1);\\\n\
    dst  = min(src2, tmp);\\\n\
} while (0)\n\
\n\
#define VXC_VertMin3_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    vxc_short8 val0_ver1, val1_ver1, val2_ver1, minVal_ver1, maxVal_ver1;\\\n\
    _viv_asm(COPY, val0_ver1, src0, 16);\\\n\
    _viv_asm(COPY, val1_ver1, src1, 16);\\\n\
    _viv_asm(COPY, val2_ver1, src2, 16);\\\n\
    maxVal_ver1 = max(val0_ver1, val1_ver1);\\\n\
    minVal_ver1 = min(val0_ver1, val1_ver1);\\\n\
    minVal_ver1 = maxVal_ver1 < 0 ? maxVal_ver1 : minVal_ver1; \\\n\
    maxVal_ver1 = max(val2_ver1, minVal_ver1);\\\n\
    minVal_ver1 = min(val2_ver1, minVal_ver1);\\\n\
    minVal_ver1 = maxVal_ver1 < 0 ? maxVal_ver1 : minVal_ver1; \\\n\
    _viv_asm(COPY, dst, minVal_ver1, 16); \\\n\
} while (0)\n\
\n\
#define VXC_VertMax3_Integer(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    constant int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    constant int endBin       = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    constant int sourceBin    = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    constant int mod1 = VXC_MODIFIER_CLAMP(startBin, endBin, sourceBin, 0);\\\n\
    typeof (dst) tmp;\\\n\
    tmp = max(src0, src1);\\\n\
    tmp = max(src2, tmp);\\\n\
    VXC_Clamp(dst, tmp, tmp, tmp, mod1);\\\n\
} while (0)\n\
\n\
#define VXC_VertMax3_Half(dst, src0, src1, src2, info)\\\n\
 do\\\n\
 {\\\n\
     vxc_short8 val0_ver1, val1_ver1, val2_ver1, minVal_ver1, maxVal_ver1;\\\n\
     _viv_asm(COPY, val0_ver1, src0, 16);\\\n\
     _viv_asm(COPY, val1_ver1, src1, 16);\\\n\
     _viv_asm(COPY, val2_ver1, src2, 16);\\\n\
     maxVal_ver1 = max(val0_ver1, val1_ver1);\\\n\
     maxVal_ver1 = max(val2_ver1, maxVal_ver1);\\\n\
     minVal_ver1 = min(val0_ver1, val1_ver1);\\\n\
     minVal_ver1 = min(val2_ver1, minVal_ver1);\\\n\
     maxVal_ver1 = maxVal_ver1 >= 0 ? maxVal_ver1 : minVal_ver1;\\\n\
     _viv_asm(COPY, dst, maxVal_ver1, 16); \\\n\
 } while (0)\n\
\n\
#define VXC_HorzMax3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    constant int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    constant int endBin       = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    constant int sourceBin    = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    constant int clamp        = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Max, clamp);\\\n\
    VXC_OP4(filter, dst, src0, src0, src0, mod1);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMax3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    constant int startBin    = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    constant int endBin      = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    constant int sourceBin   = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    constant int clamp       = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    constant int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Max, clamp);\\\n\
    constant int mod2 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Min, clamp);\\\n\
    vxc_short8 val0, minVal, maxVal;\\\n\
    _viv_asm(COPY, val0, src0, 16);\\\n\
    VXC_OP4(filter, maxVal, val0, val0, val0, mod1);\\\n\
    VXC_OP4(filter, minVal, val0, val0, val0, mod2);\\\n\
    maxVal = maxVal >= 0 ? maxVal : minVal;\\\n\
    _viv_asm(COPY, dst, maxVal, 16);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    constant int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    constant int endBin       = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    constant int sourceBin    = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    constant int clamp        = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    constant int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Min, clamp);\\\n\
    VXC_OP4(filter, dst, src0, src0, src0, mod1);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    constant int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    constant int endBin       = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    constant int sourceBin    = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    constant int clamp        = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    constant int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Max, clamp);\\\n\
    constant int mod2 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Min, clamp);\\\n\
    constant int mod3 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Median, clamp);\\\n\
    vxc_short8 val0, minVal, maxVal, midVal;\\\n\
    _viv_asm(COPY, val0, src0, 16);\\\n\
    VXC_OP4(filter, maxVal, val0, val0, val0, mod1);\\\n\
    VXC_OP4(filter, minVal, val0, val0, val0, mod2);\\\n\
    VXC_OP4(filter, midVal, val0, val0, val0, mod3);\\\n\
    minVal = midVal  < 0 ? midVal : minVal;\\\n\
    minVal = maxVal  < 0 ? maxVal : minVal;\\\n\
    _viv_asm(COPY, dst, minVal, 16);\\\n\
} while (0)\n\
\n\
#define VXC_Clamp_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    typeof (dst) tmp;\\\n\
    VXC_VertMax3_Half(tmp, src0, src0, src1, info);\\\n\
    VXC_VertMin3_Half(dst, tmp, tmp, src2, info);\\\n\
} while (0)\n\
\n\
#else\n\
#define VXC_DP2x8_b_(dst, src0, src1, src2, info, uniform)\\\n\
do\\\n\
{\\\n\
    VXC_DP2x8_b(dst, src0, src1, src2, info, uniform); \\\n\
} while (0)\n\
\n\
#define VXC_VertMin3_Integer(dst, src0, src1, src2, info)\\\n\
 do\\\n\
 {\\\n\
    VXC_VertMin3(dst, src0, src1, src2, info);\\\n\
 } while (0)\n\
\n\
#define VXC_VertMin3_Half(dst, src0, src1, src2, info)\\\n\
 do\\\n\
 {\\\n\
    VXC_VertMin3(dst, src0, src1, src2, info);\\\n\
 } while (0)\n\
\n\
#define VXC_VertMax3_Integer(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    VXC_VertMax3(dst, src0, src1, src2, info);\\\n\
} while (0)\n\
\n\
#define VXC_VertMax3_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    VXC_VertMax3(dst, src0, src1, src2, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMax3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMax3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMax3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMax3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMin3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMin3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_Clamp_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    VXC_Clamp(dst, src0, src1, src2, info);\\\n\
} while (0)\n\
#endif\n\
"; /* end of vsi_nn_kernel_header_vx*/

static const char warp_affine_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertDatatoF32_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDatatoF32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
#define WARP_AFFINE_SH_IMPL(name0, name1, src_type, src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
__kernel void warp_affine_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       matrix, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1)); \\\n\
 \\\n\
    float4 coord_f = convert_float4(coord_in); \\\n\
 \\\n\
    int2 m_coord = (int2)(0, 0); \\\n\
    vxc_ushort8 m0, m1; \\\n\
    VXC_ReadImage(m0, matrix, m_coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(m1, matrix, m_coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 matrix0, matrix1; \\\n\
    _viv_asm(COPY, matrix0, m0, 16); \\\n\
    _viv_asm(COPY, matrix1, m1, 16); \\\n\
 \\\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy; \\\n\
 \\\n\
    coord_in = convert_int4(coord_f < 0 ? coord_f - 2 : coord_f); \\\n\
 \\\n\
    int4 coord_in0 = (int4)(coord_in.xy, coord.zw); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in0.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in0.z, baseAddr); \\\n\
 \\\n\
    src_type v0, v1; \\\n\
    src_copy_type top, bot; \\\n\
    VXC_OP4(img_load_3d, v0, input, coord_in0, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, v1, input, coord_in0, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in0.xy = coord_in.zw; \\\n\
    VXC_OP4(img_load_3d, v0, input, coord_in0, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, v1, input, coord_in0, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, top, v0, 16); \\\n\
    _viv_asm(COPY, bot, v1, 16); \\\n\
 \\\n\
    float4 lerp       = coord_f - floor(coord_f); \\\n\
    float4 minus_lerp = 1.0f - lerp; \\\n\
    float4 coef0          = (float4)( minus_lerp.x * minus_lerp.y, lerp.x * minus_lerp.y, \\\n\
                                      minus_lerp.x * lerp.y,       lerp.x * lerp.y); \\\n\
 \\\n\
    float4 coef1          = (float4)( minus_lerp.z * minus_lerp.w, lerp.z * minus_lerp.w, \\\n\
                               minus_lerp.z * lerp.w,       lerp.z * lerp.w); \\\n\
 \\\n\
    float4 data0, data1, result = 0; \\\n\
    VXC_DP4x4(data0, top, bot, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(data1, top, bot, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDatatoF32_1_4x4); \\\n\
 \\\n\
    data0 = data0 * input_scale + input_tail; \\\n\
    data1 = data1 * input_scale + input_tail; \\\n\
    result.x = dot(data0, coef0); \\\n\
    result.y = dot(data1, coef1); \\\n\
    result.xy = result.xy * output_scale + output_zp; \\\n\
    convert_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 1, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 1, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
WARP_AFFINE_SH_IMPL(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
WARP_AFFINE_SH_IMPL(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
WARP_AFFINE_SH_IMPL(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
WARP_AFFINE_SH_IMPL(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
WARP_AFFINE_SH_IMPL(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
WARP_AFFINE_SH_IMPL(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
WARP_AFFINE_SH_IMPL(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
WARP_AFFINE_SH_IMPL(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
WARP_AFFINE_SH_IMPL(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
WARP_AFFINE_SH_IMPL(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
"; /* end of warp_affine_vx*/



static const char add_mean_std_norm_cl[] = "\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_F32_F32toF32(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
    float rsEps, float dimRatio,\n\
    float input0Scale, float input0Tail,\n\
    float input1Scale, float input1Tail,\n\
    float outputScale, float outputZP,\n\
    int width)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    float4 src0, src1, result;\n\
    float pSum = 0.0f, pSqr = 0.0f;\n\
    float sum  = 0.0f,  sqr = 0.0f;\n\
    float input_d = 0.0f;\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = read_imagef(input, coord);\n\
        src1 = read_imagef(input1, coord);\n\
        input_d = src0.x + src1.x;\n\
        pSum += input_d;\n\
        pSqr += input_d * input_d;\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
    float mean;\n\
    mean = sum * dimRatio;\n\
    float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = read_imagef(input, coord);\n\
        src1 = read_imagef(input1, coord);\n\
        input_d = src0.x + src1.x;\n\
        result.x = input_d * stddev_inv + rMeanStd;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_U8_U8toF32(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
    float rsEps, float dimRatio,\n\
    float input0Scale, float input0Tail,\n\
    float input1Scale, float input1Tail,\n\
    float outputScale, float outputZP,\n\
    int width)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    float4 src0, src1, result;\n\
    float pSum = 0.0f, pSqr = 0.0f;\n\
    float sum  = 0.0f,  sqr = 0.0f;\n\
    float input_d = 0.0f;\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        pSum += input_d;\n\
        pSqr += input_d * input_d;\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
    float mean;\n\
    mean = sum * dimRatio;\n\
    float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        result.x = input_d * stddev_inv + rMeanStd;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_U8_U8toU8(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
    float rsEps, float dimRatio,\n\
    float input0Scale, float input0Tail,\n\
    float input1Scale, float input1Tail,\n\
    float outputScale, float outputZP,\n\
    int width)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    float4 src0, src1, result = 0.0f;\n\
    float pSum = 0.0f, pSqr = 0.0f;\n\
    float sum  = 0.0f,  sqr = 0.0f;\n\
    float input_d = 0.0f;\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        pSum += input_d;\n\
        pSqr += input_d * input_d;\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
    float mean;\n\
    mean = sum * dimRatio;\n\
    float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        result.x = input_d * stddev_inv + rMeanStd;\n\
        uint4 dst = convert_uint4(result * outputScale + outputZP);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
"; /* end of add_mean_std_norm_cl*/

static const char argmax_axis0_cl[] = "__kernel void argmax_axis0_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
"; /* end of argmax_axis0_cl*/

static const char argmax_axis1_cl[] = "__kernel void argmax_axis1_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
"; /* end of argmax_axis1_cl*/

static const char argmax_axis2_cl[] = "__kernel void argmax_axis2_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.z : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.z : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.z : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
"; /* end of argmax_axis2_cl*/

static const char argmin_axis0_cl[] = "__kernel void argmin_axis0_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
"; /* end of argmin_axis0_cl*/

static const char argmin_axis1_cl[] = "__kernel void argmin_axis1_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
"; /* end of argmin_axis1_cl*/

static const char argmin_axis2_cl[] = "__kernel void argmin_axis2_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.z : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.z : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.z : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
"; /* end of argmin_axis2_cl*/

static const char avg_pool3d_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define TENSOR_AVG_POOL3D(src_name, dst_name, src_type, dst_type,\\\n\
                         readimage_type, conv_mode, writeimage_type) \\\n\
__kernel void avg_pool3d_##src_name##to##dst_name ( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              ksize_x, \\\n\
                 int              ksize_y, \\\n\
                 int              ksize_z, \\\n\
                 int              stride_x, \\\n\
                 int              stride_y, \\\n\
                 int              stride_z, \\\n\
                 int              pad_left, \\\n\
                 int              pad_top, \\\n\
                 int              pad_front, \\\n\
                 int              width, \\\n\
                 int              height, \\\n\
                 int              depth_in, \\\n\
                 int              depth_out, \\\n\
                 float            inputScale, \\\n\
                 float            inputTail, \\\n\
                 float            outputScale, \\\n\
                 float            outputTail, \\\n\
                 int              count_include_pad) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int offsetz = get_global_id(2); \\\n\
    int offsetz2 = offsetz / depth_out * depth_in; \\\n\
    int d, d2, h, w, count; \\\n\
    float sum = 0; \\\n\
    dst_type out_data = (dst_type)(0); \\\n\
    src_type in_data; \\\n\
    float in_f32, out_f32; \\\n\
    int wstart = gidx * stride_x - pad_left; \\\n\
    int hstart = gidy * stride_y - pad_top; \\\n\
    int wend = min(wstart + ksize_x, width); \\\n\
    int hend = min(hstart + ksize_y, height); \\\n\
    int dstart, dend; \\\n\
    int4 coord_in, coord_out; \\\n\
    wstart = max(wstart, 0); \\\n\
    hstart = max(hstart, 0); \\\n\
    for (d2 = 0; d2 < depth_out; d2++) \\\n\
    { \\\n\
        dstart = d2 * stride_z - pad_front; \\\n\
        dend = min(dstart + ksize_z, depth_in); \\\n\
        dstart = max(dstart, 0); \\\n\
        coord_out = (int4)(gidx, gidy, offsetz + d2, 0); \\\n\
        sum = 0; \\\n\
        count = 0; \\\n\
        for (d = dstart; d < dend; d++) \\\n\
        { \\\n\
            for (h = hstart; h < hend; h++) \\\n\
            { \\\n\
                for (w = wstart; w < wend; w++) \\\n\
                { \\\n\
                    coord_in = (int4)(w, h, d + offsetz2, 0); \\\n\
                    in_data = readimage_type(input, coord_in).x; \\\n\
                    in_f32 = convert_float(in_data) * inputScale + inputTail; \\\n\
                    sum += in_f32; \\\n\
                    count++; \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
        if (count_include_pad == 1) \\\n\
        { \\\n\
            count = ksize_x * ksize_y * ksize_z; \\\n\
        } \\\n\
        out_f32 = (sum / count) * outputScale + outputTail; \\\n\
        out_data.x = conv_mode(out_f32); \\\n\
        writeimage_type(output, coord_out, out_data); \\\n\
    } \\\n\
}\n\
\n\
TENSOR_AVG_POOL3D(F32, F32, float, float4, read_imagef, convert_float, write_imagef)\n\
TENSOR_AVG_POOL3D(F32, U32, float, uint4,  read_imagef, convert_uint,  write_imageui)\n\
TENSOR_AVG_POOL3D(F32, I32, float, int4,   read_imagef, convert_int,   write_imagei)\n\
\n\
TENSOR_AVG_POOL3D(U32, U32, uint, uint4,  read_imageui, convert_uint,  write_imageui)\n\
TENSOR_AVG_POOL3D(U32, F32, uint, float4, read_imageui, convert_float, write_imagef)\n\
TENSOR_AVG_POOL3D(U32, I32, uint, int4,   read_imageui, convert_int,   write_imagei)\n\
\n\
TENSOR_AVG_POOL3D(I32, I32, int, int4,    read_imagei, convert_int,   write_imagei)\n\
TENSOR_AVG_POOL3D(I32, F32, int, float4, read_imagei, convert_float, write_imagef)\n\
TENSOR_AVG_POOL3D(I32, U32, int, uint4,  read_imagei, convert_uint,  write_imageui)\n\
\n\
__kernel void avg_pool3d_BF16toBF16 (\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
                 int              ksize_x,\n\
                 int              ksize_y,\n\
                 int              ksize_z,\n\
                 int              stride_x,\n\
                 int              stride_y,\n\
                 int              stride_z,\n\
                 int              pad_left,\n\
                 int              pad_top,\n\
                 int              pad_front,\n\
                 int              width,\n\
                 int              height,\n\
                 int              depth_in,\n\
                 int              depth_out,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputTail,\n\
                 int              count_include_pad)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int offsetz = get_global_id(2);\n\
    int offsetz2 = offsetz / depth_out * depth_in;\n\
    int d, d2, h, w, count;\n\
    float sum = 0;\n\
    uint4 out_data = (uint4)(0);\n\
    uint4 in_data;\n\
    float in_f32, out_f32;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wend = min(wstart + ksize_x, width);\n\
    int hend = min(hstart + ksize_y, height);\n\
    int dstart, dend;\n\
    int4 coord_in, coord_out;\n\
    wstart = max(wstart, 0);\n\
    hstart = max(hstart, 0);\n\
    for (d2 = 0; d2 < depth_out; d2++)\n\
    {\n\
        dstart = d2 * stride_z - pad_front;\n\
        dend = min(dstart + ksize_z, depth_in);\n\
        dstart = max(dstart, 0);\n\
        coord_out = (int4)(gidx, gidy, offsetz + d2, 0);\n\
        sum = 0;\n\
        count = 0;\n\
        for (d = dstart; d < dend; d++)\n\
        {\n\
            for (h = hstart; h < hend; h++)\n\
            {\n\
                for (w = wstart; w < wend; w++)\n\
                {\n\
                    coord_in = (int4)(w, h, d + offsetz2, 0);\n\
                    in_data = read_imageui(input, coord_in).x;\n\
                    in_data = in_data << 16;\n\
                    _viv_asm(COPY, in_f32, in_data, 16);\n\
                    sum += in_f32;\n\
                    count++;\n\
                }\n\
            }\n\
        }\n\
        if (count_include_pad == 1)\n\
        {\n\
            count = ksize_x * ksize_y * ksize_z;\n\
        }\n\
        out_f32 = sum / count;\n\
        _viv_asm(COPY, out_data, out_f32, 4);\n\
        out_data.x = out_data.x >> 16;\n\
        write_imageui(output, coord_out, out_data);\n\
    }\n\
}"; /* end of avg_pool3d_cl*/

static const char batchnorm_single_cl[] = "#define BN_U8_SAVE \\\n\
    uint4 dst = convert_uint4(src * output_scale + output_zp); \\\n\
    write_imageui(output, coord, dst);\n\
\n\
#define BN_I32_SAVE \\\n\
    int4 dst = convert_int4(src * output_scale + output_zp); \\\n\
    write_imagei(output, coord, dst);\n\
\n\
#define BN_F32_SAVE \\\n\
    write_imagef(output, coord, src);\n\
\n\
#define BATCH_NORM_F32_SH_IMPL(TYPE) \\\n\
__kernel void batch_norm_F32to##TYPE \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 float           output_scale, \\\n\
                 float           output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src, mean, var, gamma, beta; \\\n\
    READ_IMAGEF_2DARRAY(src, input, coord); \\\n\
    READ_IMAGEF_2DARRAY(mean, Mean, coord); \\\n\
    READ_IMAGEF_2DARRAY(var, Variance, coord); \\\n\
    READ_IMAGEF_2DARRAY(gamma, Gamma, coord); \\\n\
    READ_IMAGEF_2DARRAY(beta, Beta, coord); \\\n\
 \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_F32_SH_IMPL(F32)\n\
BATCH_NORM_F32_SH_IMPL(U8)\n\
BATCH_NORM_F32_SH_IMPL(I32)\n\
\n\
#define BATCH_NORM_F32_SH_IMPL_2D(TYPE) \\\n\
__kernel void batch_norm_F32to##TYPE##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t Mean, \\\n\
    __read_only  image2d_t Variance, \\\n\
    __read_only  image2d_t Gamma, \\\n\
    __read_only  image2d_t Beta, \\\n\
    __write_only image2d_t output, \\\n\
                 float     eps, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 float     output_scale, \\\n\
                 float     output_zp \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
    float4 mean = read_imagef(Mean, coord); \\\n\
    float4 var = read_imagef(Variance, coord); \\\n\
    float4 gamma = read_imagef(Gamma, coord); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_F32_SH_IMPL_2D(F32)\n\
BATCH_NORM_F32_SH_IMPL_2D(U8)\n\
BATCH_NORM_F32_SH_IMPL_2D(I32)\n\
\n\
#define BATCH_NORM_U8_SH_IMPL(TYPE) \\\n\
__kernel void batch_norm_U8to##TYPE \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 float           output_scale, \\\n\
                 float           output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 data; \\\n\
    float4 src, mean, var, gamma, beta; \\\n\
    READ_IMAGEUI_2DARRAY(data, input, coord); \\\n\
    READ_IMAGEF_2DARRAY(mean, Mean, coord); \\\n\
    READ_IMAGEF_2DARRAY(var, Variance, coord); \\\n\
    READ_IMAGEF_2DARRAY(gamma, Gamma, coord); \\\n\
    READ_IMAGEF_2DARRAY(beta, Beta, coord); \\\n\
 \\\n\
    src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_U8_SH_IMPL(U8)\n\
BATCH_NORM_U8_SH_IMPL(F32)\n\
\n\
#define BATCH_NORM_U8_SH_IMPL_2D(TYPE) \\\n\
__kernel void batch_norm_U8to##TYPE##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t Mean, \\\n\
    __read_only  image2d_t Variance, \\\n\
    __read_only  image2d_t Gamma, \\\n\
    __read_only  image2d_t Beta, \\\n\
    __write_only image2d_t output, \\\n\
                 float     eps, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 float     output_scale, \\\n\
                 float     output_zp \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4  data = read_imageui(input, coord); \\\n\
    float4 mean = read_imagef(Mean, coord); \\\n\
    float4 var = read_imagef(Variance, coord); \\\n\
    float4 gamma = read_imagef(Gamma, coord); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    float4 src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_U8_SH_IMPL_2D(U8)\n\
BATCH_NORM_U8_SH_IMPL_2D(F32)\n\
\n\
#define BATCH_NORM_I32_SH_IMPL(TYPE) \\\n\
__kernel void batch_norm_I32to##TYPE \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 float           output_scale, \\\n\
                 float           output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    int4 data; \\\n\
    float4 src, mean, var, gamma, beta; \\\n\
    READ_IMAGEI_2DARRAY(data, input, coord); \\\n\
    READ_IMAGEF_2DARRAY(mean, Mean, coord); \\\n\
    READ_IMAGEF_2DARRAY(var, Variance, coord); \\\n\
    READ_IMAGEF_2DARRAY(gamma, Gamma, coord); \\\n\
    READ_IMAGEF_2DARRAY(beta, Beta, coord); \\\n\
 \\\n\
    src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_I32_SH_IMPL(I32)\n\
BATCH_NORM_I32_SH_IMPL(F32)\n\
\n\
#define BATCH_NORM_I32_SH_IMPL_2D(TYPE) \\\n\
__kernel void batch_norm_I32to##TYPE##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t Mean, \\\n\
    __read_only  image2d_t Variance, \\\n\
    __read_only  image2d_t Gamma, \\\n\
    __read_only  image2d_t Beta, \\\n\
    __write_only image2d_t output, \\\n\
                 float     eps, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 float     output_scale, \\\n\
                 float     output_zp \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    int4  data = read_imagei(input, coord); \\\n\
    float4 mean = read_imagef(Mean, coord); \\\n\
    float4 var = read_imagef(Variance, coord); \\\n\
    float4 gamma = read_imagef(Gamma, coord); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    float4 src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_I32_SH_IMPL_2D(I32)\n\
BATCH_NORM_I32_SH_IMPL_2D(F32)"; /* end of batchnorm_single_cl*/

static const char bilinear_grid_sample_cl[] = "__kernel void bilinear_grid_sample_F32_F32toF32(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                           float  half_input0_w,\n\
                           float  half_input0_h,\n\
                           float  add_float_value_w,\n\
                           float  add_float_value_h,\n\
                           int    depth\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int2   coord_in1    =  (int2)(get_global_id(0) * 2, get_global_id(1));\n\
    int2   coord_add    = (int2)(-1, 1);\n\
\n\
    float fx = read_imagef(input1, coord_in1).x;\n\
    coord_in1.x = coord_in1.x + 1;\n\
    float fy = read_imagef(input1, coord_in1).x;\n\
\n\
    fx = fx * half_input0_w + add_float_value_w;\n\
    fy = fy * half_input0_h + add_float_value_h;\n\
    float x_f = floor(fx);\n\
    float y_f = floor(fy);\n\
    float x_lerp  = fx - x_f;\n\
    float y_lerp  = fy - y_f;\n\
    int   x_index = convert_int(x_f);\n\
    int   y_index = convert_int(y_f);\n\
    int4   coord_in     = (int4)(x_index, y_index, 0, 0);\n\
\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom, dst;\n\
\n\
    while (coord_in.z < depth){\n\
        top_l    = read_imagef(input0, coord_in);\n\
        coord_in.y++;\n\
        bottom_l = read_imagef(input0, coord_in);\n\
        coord_in.x++;\n\
        bottom_r = read_imagef(input0, coord_in);\n\
        coord_in.y--;\n\
        top_r    = read_imagef(input0, coord_in);\n\
        top_r    = top_r - top_l;\n\
        top      = top_l + x_lerp * top_r;\n\
        bottom_r = bottom_r - bottom_l;\n\
        bottom   = bottom_l + x_lerp * bottom_r;\n\
        bottom   = bottom - top;\n\
        dst      = top + y_lerp * bottom;\n\
        write_imagef(output, coord_out, dst);\n\
        coord_in.xz = coord_in.xz + coord_add;\n\
        coord_out.z++;\n\
    }\n\
}\n\
\n\
\n\
__kernel void bilinear_grid_sample_U8_U8toU8(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                           float  half_input0_w,\n\
                           float  half_input0_h,\n\
                           float  add_float_value_w,\n\
                           float  add_float_value_h,\n\
                           int    depth,\n\
                           float  in0_scale,\n\
                           float  in0_tail,\n\
                           float  in1_scale,\n\
                           float  in1_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int2   coord_in1    =  (int2)(get_global_id(0) * 2, get_global_id(1));\n\
    int2   coord_add    = (int2)(-1, 1);\n\
\n\
    float fx    = convert_float4(read_imageui(input1, coord_in1)).x * in1_scale + in1_tail;\n\
    coord_in1.x = coord_in1.x + 1;\n\
    float fy    = convert_float4(read_imageui(input1, coord_in1)).x * in1_scale + in1_tail;\n\
\n\
    fx = fx * half_input0_w + add_float_value_w;\n\
    fy = fy * half_input0_h + add_float_value_h;\n\
    float x_f = floor(fx);\n\
    float y_f = floor(fy);\n\
    float x_lerp  = fx - x_f;\n\
    float y_lerp  = fy - y_f;\n\
    int   x_index = convert_int(x_f);\n\
    int   y_index = convert_int(y_f);\n\
    int4   coord_in     = (int4)(x_index, y_index, 0, 0);\n\
\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom;\n\
    uint4  dst;\n\
\n\
    while (coord_in.z < depth){\n\
        top_l    = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        coord_in.y++;\n\
        bottom_l = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        coord_in.x++;\n\
        bottom_r = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        coord_in.y--;\n\
        top_r    = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        top_r    = top_r - top_l;\n\
        top      = top_l + x_lerp * top_r;\n\
        bottom_r = bottom_r - bottom_l;\n\
        bottom   = bottom_l + x_lerp * bottom_r;\n\
        bottom   = bottom - top;\n\
        top      = top + y_lerp * bottom;\n\
        dst      = convert_uint4_rte(top * out_scale + out_tail);\n\
        write_imageui(output, coord_out, dst);\n\
        coord_in.xz = coord_in.xz + coord_add;\n\
        coord_out.z++;\n\
    }\n\
\n\
}"; /* end of bilinear_grid_sample_cl*/

static const char bilinear_grid_sample_reflect_cl[] = "__kernel void bilinear_grid_sample_reflect_F32_F32toF32(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                           float  half_input0_w,\n\
                           float  half_input0_h,\n\
                           float  add_float_value_w,\n\
                           float  add_float_value_h,\n\
                           int    depth,\n\
                           float  min_val_w,\n\
                           float  span_w,\n\
                           float  min_val_h,\n\
                           float  span_h\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int2   coord_in1    =  (int2)(get_global_id(0) * 2, get_global_id(1));\n\
    int2   coord_add    = (int2)(-1, 1);\n\
\n\
    float fx = read_imagef(input1, coord_in1).x;\n\
    coord_in1.x = coord_in1.x + 1;\n\
    float fy = read_imagef(input1, coord_in1).x;\n\
\n\
    fx = fx * half_input0_w + add_float_value_w;\n\
    fy = fy * half_input0_h + add_float_value_h;\n\
\n\
    if (span_w > 0)\n\
    {\n\
        fx = fabs(fx - min_val_w);\n\
        int flips_x = (int)(fx / span_w);\n\
        float extra_x = fx - flips_x * span_w;\n\
        fx = (flips_x & 0x01) ?  min_val_w + (span_w - extra_x) : min_val_w + extra_x ;\n\
    }\n\
    else\n\
    {\n\
        fx = 0;\n\
    }\n\
\n\
    if (span_h > 0)\n\
    {\n\
        fy = fabs(fy - min_val_h);\n\
        int flips_y = (int)(fy / span_h);\n\
        float extra_y = fy - flips_y * span_h;\n\
        fy = (flips_y & 0x01) ?  min_val_h + (span_h - extra_y) : min_val_h + extra_y ;\n\
    }\n\
    else\n\
    {\n\
        fy = 0;\n\
    }\n\
\n\
    float x_f = floor(fx);\n\
    float y_f = floor(fy);\n\
    float x_lerp  = fx - x_f;\n\
    float y_lerp  = fy - y_f;\n\
    int   x_index = convert_int(x_f);\n\
    int   y_index = convert_int(y_f);\n\
    int4   coord_in     = (int4)(x_index, y_index, 0, 0);\n\
\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom, dst;\n\
\n\
    while (coord_in.z < depth){\n\
        top_l    = read_imagef(input0, coord_in);\n\
        coord_in.y++;\n\
        bottom_l = read_imagef(input0, coord_in);\n\
        coord_in.x++;\n\
        bottom_r = read_imagef(input0, coord_in);\n\
        coord_in.y--;\n\
        top_r    = read_imagef(input0, coord_in);\n\
        top_r    = top_r - top_l;\n\
        top      = top_l + x_lerp * top_r;\n\
        bottom_r = bottom_r - bottom_l;\n\
        bottom   = bottom_l + x_lerp * bottom_r;\n\
        bottom   = bottom - top;\n\
        dst      = top + y_lerp * bottom;\n\
        write_imagef(output, coord_out, dst);\n\
        coord_in.xz = coord_in.xz + coord_add;\n\
        coord_out.z++;\n\
    }\n\
}\n\
\n\
\n\
__kernel void bilinear_grid_sample_reflect_U8_U8toU8(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                           float  half_input0_w,\n\
                           float  half_input0_h,\n\
                           float  add_float_value_w,\n\
                           float  add_float_value_h,\n\
                           int    depth,\n\
                           float  in0_scale,\n\
                           float  in0_tail,\n\
                           float  in1_scale,\n\
                           float  in1_tail,\n\
                           float  out_scale,\n\
                           float  out_tail,\n\
                           float  min_val_w,\n\
                           float  span_w,\n\
                           float  min_val_h,\n\
                           float  span_h\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int2   coord_in1    =  (int2)(get_global_id(0) * 2, get_global_id(1));\n\
    int2   coord_add    = (int2)(-1, 1);\n\
\n\
    float fx    = convert_float4(read_imageui(input1, coord_in1)).x * in1_scale + in1_tail;\n\
    coord_in1.x = coord_in1.x + 1;\n\
    float fy    = convert_float4(read_imageui(input1, coord_in1)).x * in1_scale + in1_tail;\n\
\n\
    fx = fx * half_input0_w + add_float_value_w;\n\
    fy = fy * half_input0_h + add_float_value_h;\n\
\n\
    if (span_w > 0)\n\
    {\n\
        fx = fabs(fx - min_val_w);\n\
        int flips_x = (int)(fx / span_w);\n\
        float extra_x = fx - flips_x * span_w;\n\
        fx = (flips_x & 0x01) ?  min_val_w + (span_w - extra_x) : min_val_w + extra_x ;\n\
    }\n\
    else\n\
    {\n\
        fx = 0;\n\
    }\n\
\n\
    if (span_h > 0)\n\
    {\n\
        fy = fabs(fy - min_val_h);\n\
        int flips_y = (int)(fy / span_h);\n\
        float extra_y = fy - flips_y * span_h;\n\
        fy = (flips_y & 0x01) ?  min_val_h + (span_h - extra_y) : min_val_h + extra_y ;\n\
    }\n\
    else\n\
    {\n\
        fy = 0;\n\
    }\n\
\n\
    float x_f = floor(fx);\n\
    float y_f = floor(fy);\n\
    float x_lerp  = fx - x_f;\n\
    float y_lerp  = fy - y_f;\n\
    int   x_index = convert_int(x_f);\n\
    int   y_index = convert_int(y_f);\n\
    int4   coord_in     = (int4)(x_index, y_index, 0, 0);\n\
\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom;\n\
    uint4  dst;\n\
\n\
    while (coord_in.z < depth){\n\
        top_l    = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        coord_in.y++;\n\
        bottom_l = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        coord_in.x++;\n\
        bottom_r = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        coord_in.y--;\n\
        top_r    = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        top_r    = top_r - top_l;\n\
        top      = top_l + x_lerp * top_r;\n\
        bottom_r = bottom_r - bottom_l;\n\
        bottom   = bottom_l + x_lerp * bottom_r;\n\
        bottom   = bottom - top;\n\
        top      = top + y_lerp * bottom;\n\
        dst      = convert_uint4_rte(top * out_scale + out_tail);\n\
        write_imageui(output, coord_out, dst);\n\
        coord_in.xz = coord_in.xz + coord_add;\n\
        coord_out.z++;\n\
    }\n\
\n\
}"; /* end of bilinear_grid_sample_reflect_cl*/

static const char bucketize_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
\n\
#define BUCKETIZE_F32_2D_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t boundaries, \\\n\
    __write_only image2d_t output, \\\n\
                 int       boundaries_size, \\\n\
                 float     input0_scale, \\\n\
                 float     input0_tail, \\\n\
                 float     input1_scale, \\\n\
                 float     input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src0 = read_imagef(input, coord); \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    do \\\n\
    { \\\n\
        float4 src1 = read_imagef(boundaries, pos); \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_F32_2D_SH_IMPL(F32_F32toI32_2D,       <=)\n\
BUCKETIZE_F32_2D_SH_IMPL(right_F32_F32toI32_2D, <)\n\
\n\
#define BUCKETIZE_F32_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       boundaries, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             boundaries_size, \\\n\
                 float           input0_scale, \\\n\
                 float           input0_tail, \\\n\
                 float           input1_scale, \\\n\
                 float           input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src0 = read_imagef(input, coord); \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    do \\\n\
    { \\\n\
        float4 src1 = read_imagef(boundaries, pos); \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_F32_SH_IMPL(F32_F32toI32,       <=)\n\
BUCKETIZE_F32_SH_IMPL(right_F32_F32toI32, <)\n\
\n\
#define BUCKETIZE_I32_2D_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t boundaries, \\\n\
    __write_only image2d_t output, \\\n\
                 int       boundaries_size, \\\n\
                 float     input0_scale, \\\n\
                 float     input0_tail, \\\n\
                 float     input1_scale, \\\n\
                 float     input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src0 = convert_float4(read_imagei(input, coord)); \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    src0 = src0 * input0_scale + input0_tail; \\\n\
    do \\\n\
    { \\\n\
        float4 src1 = convert_float4(read_imagei(boundaries, pos)); \\\n\
        src1 = src1 * input1_scale + input1_tail; \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_I32_2D_SH_IMPL(I32_I32toI32_2D,       <=)\n\
BUCKETIZE_I32_2D_SH_IMPL(right_I32_I32toI32_2D, <)\n\
\n\
#define BUCKETIZE_I32_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       boundaries, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             boundaries_size, \\\n\
                 float           input0_scale, \\\n\
                 float           input0_tail, \\\n\
                 float           input1_scale, \\\n\
                 float           input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    int4 data = read_imagei(input, coord); \\\n\
    float4 src0 = convert_float4(data) * input0_scale + input0_tail; \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    do \\\n\
    { \\\n\
        float4 src1 = convert_float4(read_imagei(boundaries, pos)); \\\n\
        src1 = src1 * input1_scale + input1_tail; \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_I32_SH_IMPL(I32_I32toI32,       <=)\n\
BUCKETIZE_I32_SH_IMPL(right_I32_I32toI32, <)\n\
\n\
#define BUCKETIZE_U32_2D_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t boundaries, \\\n\
    __write_only image2d_t output, \\\n\
                 int       boundaries_size, \\\n\
                 float     input0_scale, \\\n\
                 float     input0_tail, \\\n\
                 float     input1_scale, \\\n\
                 float     input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src0 = convert_float4(read_imageui(input, coord)); \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    src0 = src0 * input0_scale + input0_tail; \\\n\
    do \\\n\
    { \\\n\
        float4 src1 = convert_float4(read_imageui(boundaries, pos)); \\\n\
        src1 = src1 * input1_scale + input1_tail; \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_U32_2D_SH_IMPL(U32_U32toI32_2D,       <=)\n\
BUCKETIZE_U32_2D_SH_IMPL(right_U32_U32toI32_2D, <)\n\
\n\
#define BUCKETIZE_U32_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       boundaries, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             boundaries_size, \\\n\
                 float           input0_scale, \\\n\
                 float           input0_tail, \\\n\
                 float           input1_scale, \\\n\
                 float           input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 data = read_imageui(input, coord); \\\n\
    float4 src0 = convert_float4(data) * input0_scale + input0_tail; \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    do \\\n\
    { \\\n\
        float4 src1 = convert_float4(read_imageui(boundaries, pos)); \\\n\
        src1 = src1 * input1_scale + input1_tail; \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_U32_SH_IMPL(U32_U32toI32,       <=)\n\
BUCKETIZE_U32_SH_IMPL(right_U32_U32toI32, <)\n\
\n\
#define BUCKETIZE_BF16_2D_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t boundaries, \\\n\
    __write_only image2d_t output, \\\n\
                 int       boundaries_size, \\\n\
                 float     input0_scale, \\\n\
                 float     input0_tail, \\\n\
                 float     input1_scale, \\\n\
                 float     input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 data0 = read_imageui(input, coord) << 16; \\\n\
    float4 src0; \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    do \\\n\
    { \\\n\
        uint4 data1 = read_imageui(boundaries, pos) << 16; \\\n\
        float4 src1; \\\n\
        _viv_asm(COPY, src1, data1, 16); \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_BF16_2D_SH_IMPL(BF16_BF16toI32_2D,       <=)\n\
BUCKETIZE_BF16_2D_SH_IMPL(right_BF16_BF16toI32_2D, <)\n\
\n\
#define BUCKETIZE_BF16_SH_IMPL(name, comp_op) \\\n\
__kernel void bucketize_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       boundaries, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             boundaries_size, \\\n\
                 float           input0_scale, \\\n\
                 float           input0_tail, \\\n\
                 float           input1_scale, \\\n\
                 float           input1_tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 data0 = read_imageui(input, coord) << 16; \\\n\
    float4 src0; \\\n\
    _viv_asm(COPY, src0, data0, 16); \\\n\
 \\\n\
    int2 pos = 0; \\\n\
    do \\\n\
    { \\\n\
        uint4 data1 = read_imageui(boundaries, pos) << 16; \\\n\
        float4 src1; \\\n\
        _viv_asm(COPY, src1, data1, 16); \\\n\
        if ((src0.x) comp_op (src1.x)) \\\n\
        { \\\n\
            break; \\\n\
        } \\\n\
        pos.x ++; \\\n\
    } while(pos.x < boundaries_size); \\\n\
 \\\n\
    write_imagei(output, coord, pos.xxxx); \\\n\
}\n\
BUCKETIZE_BF16_SH_IMPL(BF16_BF16toI32,       <=)\n\
BUCKETIZE_BF16_SH_IMPL(right_BF16_BF16toI32, <)\n\
"; /* end of bucketize_cl*/

static const char cast_cl[] = "\n\
#define CAST_FUN(src_name, dst_name, src_type, dst_type, conv_fun, read_fun, write_fun) \\\n\
__kernel void cast_##src_name##to##dst_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    dst_type dst = 0; \\\n\
    dst = conv_fun(src); \\\n\
    write_fun(output, coord, dst); \\\n\
}\n\
\n\
CAST_FUN(F32, I32, float4, int4,   convert_int4_sat,  read_imagef,  write_imagei)\n\
CAST_FUN(F32, U32, float4, uint4,  convert_uint4_sat, read_imagef,  write_imageui)\n\
CAST_FUN(I32, I32, int4,   int4,   convert_int4_sat,  read_imagei,  write_imagei)\n\
CAST_FUN(I32, U32, int4,   uint4,  convert_uint4_sat, read_imagei,  write_imageui)\n\
CAST_FUN(U32, I32, uint4,  int4,   convert_int4_sat,  read_imageui, write_imagei)\n\
CAST_FUN(U32, U32, uint4,  uint4,  convert_uint4_sat, read_imageui, write_imageui)\n\
CAST_FUN(F32, F32, float4, float4, convert_float4,    read_imagef,  write_imagef)\n\
CAST_FUN(I32, F32, int4,   float4, convert_float4,    read_imagei,  write_imagef)\n\
CAST_FUN(U32, F32, uint4,  float4, convert_float4,    read_imageui, write_imagef)\n\
\n\
#define CAST_FUN_2D(src_name, dst_name, src_type, dst_type, conv_fun, read_fun, write_fun) \\\n\
__kernel void cast_##src_name##to##dst_name##_2D( \\\n\
    __read_only  image2d_t  input, \\\n\
    __write_only image2d_t  output) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    dst_type dst = 0; \\\n\
    dst = conv_fun(src); \\\n\
    write_fun(output, coord, dst); \\\n\
}\n\
\n\
CAST_FUN_2D(F32, I32, float4, int4,   convert_int4_sat,  read_imagef,  write_imagei)\n\
CAST_FUN_2D(F32, U32, float4, uint4,  convert_uint4_sat, read_imagef,  write_imageui)\n\
CAST_FUN_2D(I32, I32, int4,   int4,   convert_int4_sat,  read_imagei,  write_imagei)\n\
CAST_FUN_2D(I32, U32, int4,   uint4,  convert_uint4_sat, read_imagei,  write_imageui)\n\
CAST_FUN_2D(U32, I32, uint4,  int4,   convert_int4_sat,  read_imageui, write_imagei)\n\
CAST_FUN_2D(U32, U32, uint4,  uint4,  convert_uint4_sat, read_imageui, write_imageui)\n\
CAST_FUN_2D(F32, F32, float4, float4, convert_float4,    read_imagef,  write_imagef)\n\
CAST_FUN_2D(I32, F32, int4,   float4, convert_float4,    read_imagei,  write_imagef)\n\
CAST_FUN_2D(U32, F32, uint4,  float4, convert_float4,    read_imageui, write_imagef)\n\
\n\
#define CAST_TO_BOOL_FUN(src_name, src_type, read_fun) \\\n\
__kernel void cast_##src_name##toBOOL8( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    int4 dst = 0; \\\n\
    dst.x = (int)(src.x != 0); \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN(F32, float4, read_imagef)\n\
CAST_TO_BOOL_FUN(I32, int4,   read_imagei)\n\
CAST_TO_BOOL_FUN(U32, uint4,  read_imageui)\n\
\n\
\n\
#define CAST_TO_BOOL_FUN_2D(src_name, src_type, read_fun) \\\n\
__kernel void cast_##src_name##toBOOL8_2D( \\\n\
    __read_only  image2d_t  input, \\\n\
    __write_only image2d_t  output) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    int4 dst = 0; \\\n\
    dst.x = (int)(src.x != 0); \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN_2D(F32, float4, read_imagef)\n\
CAST_TO_BOOL_FUN_2D(I32, int4,   read_imagei)\n\
CAST_TO_BOOL_FUN_2D(U32, uint4,  read_imageui)\n\
\n\
"; /* end of cast_cl*/

static const char clip_BF16_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
\n\
__kernel void clip_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           float minData,\n\
                           float maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    uint4 src0 = read_imageui(input, coord);\n\
    src0 = src0 << 16;\n\
    float4 src;\n\
    _viv_asm(COPY, src, src0, 16);\n\
    float4 dst0 = clamp(src, minData, maxData);\n\
    uint4 dst;\n\
    _viv_asm(COPY, dst, dst0, 16);\n\
    dst = dst >> 16;\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     float minData,\n\
                     float maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    uint4 src0 = read_imageui(input, coord);\n\
    src0 = src0 << 16;\n\
    float4 src;\n\
    _viv_asm(COPY, src, src0, 16);\n\
    float4 dst0 = clamp(src, minData, maxData);\n\
    uint4 dst;\n\
    _viv_asm(COPY, dst, dst0, 16);\n\
    dst = dst >> 16;\n\
    write_imageui(output, coord, dst);\n\
}\n\
"; /* end of clip_BF16_cl*/

static const char clip_F32_cl[] = "__kernel void clip_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           float minData,\n\
                           float maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = clamp(src, minData, maxData);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     float minData,\n\
                     float maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = clamp(src, minData, maxData);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           float minData,\n\
                           float maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = clamp(src, minData, maxData);\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     float minData,\n\
                     float maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = clamp(src, minData, maxData);\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           float minData,\n\
                           float maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = clamp(src, minData, maxData);\n\
    int4 dst = convert_int4_rte(result * outputScale + outputZP);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     float minData,\n\
                     float maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = clamp(src, minData, maxData);\n\
    int4 dst = convert_int4_rte(result * outputScale + outputZP);\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of clip_F32_cl*/

static const char clip_I32_cl[] = "__kernel void clip_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           float minData,\n\
                           float maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imagei(input, coord)) * inputScale + inputTail;\n\
    float4 result = clamp(src, minData, maxData);\n\
    int4 dst = convert_int4_rte(result * outputScale + outputZP);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     float minData,\n\
                     float maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imagei(input, coord)) * inputScale + inputTail;\n\
    float4 result = clamp(src, minData, maxData);\n\
    int4 dst = convert_int4_rte(result * outputScale + outputZP);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_I32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           float minData,\n\
                           float maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imagei(input, coord)) * inputScale + inputTail;\n\
    float4 dst = clamp(src, minData, maxData);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_I32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     float minData,\n\
                     float maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imagei(input, coord)) * inputScale + inputTail;\n\
    float4 dst = clamp(src, minData, maxData);\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of clip_I32_cl*/

static const char clip_U8_cl[] = "__kernel void clip_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src > minData ? src : minData;\n\
    result = result < maxData ? result : maxData;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_U8toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  minData,\n\
                     float  maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src > minData ? src : minData;\n\
    result = result < maxData ? result : maxData;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_U8toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src > minData ? src : minData;\n\
    dst = dst < maxData ? dst : maxData;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_U8toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  minData,\n\
                     float  maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src > minData ? src : minData;\n\
    dst = dst < maxData ? dst : maxData;\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of clip_U8_cl*/

static const char col2im_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width_pad;\n\
_viv_uniform int height_pad;\n\
_viv_uniform int depth_pad;\n\
_viv_uniform int move_time_x;\n\
_viv_uniform int move_time_y;\n\
_viv_uniform int kernel_x_new;\n\
_viv_uniform int kernel_y_new;\n\
_viv_uniform int kernel_z_new;\n\
_viv_uniform int depth;\n\
\n\
#define COL2IM(name, read_type, dst_type ,convert_type, write_type) \\\n\
__kernel void col2im_##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride_w, \\\n\
                 int              stride_h, \\\n\
                 int              stride_d, \\\n\
                 int              dilation_w, \\\n\
                 int              dilation_h, \\\n\
                 int              dilation_d, \\\n\
                 int              pad_w_front, \\\n\
                 int              pad_w_end, \\\n\
                 int              pad_h_front, \\\n\
                 int              pad_h_end, \\\n\
                 int              pad_d_front, \\\n\
                 int              pad_d_end, \\\n\
                 int              kernel_x, \\\n\
                 int              kernel_y, \\\n\
                 int              kernel_z, \\\n\
                 float            inOutScale, \\\n\
                 float            inOutTile \\\n\
) \\\n\
{ \\\n\
    int x = get_global_id(0); \\\n\
    int y = get_global_id(1); \\\n\
    int z = get_global_id(2); \\\n\
    int4 coord_out = (int4)(x,y,z,0); \\\n\
    int b = z / depth; \\\n\
    z = z % depth; \\\n\
    int4 coord_in = (int4)(0,0,b,0); \\\n\
 \\\n\
    float sum = 0.0f; \\\n\
    x = x + pad_w_front; \\\n\
    y = y + pad_h_front; \\\n\
    z = z + pad_d_front; \\\n\
    int offset_x = x % stride_w; \\\n\
    int offset_y = y % stride_h; \\\n\
    int offset_z = z % stride_d; \\\n\
    int i,j,k; \\\n\
    for (k = offset_z; k < kernel_z_new; k += stride_d) \\\n\
    { \\\n\
        if ((z - k) < 0 || (z + (kernel_z_new - k)) > depth_pad || k % dilation_d != 0) \\\n\
        { \\\n\
            continue; \\\n\
        } \\\n\
        for (j = offset_y; j < kernel_y_new; j = j + stride_h) \\\n\
        { \\\n\
            if ((y - j) < 0 || (y + (kernel_y_new - j)) > height_pad || j % dilation_h != 0) \\\n\
            { \\\n\
                continue; \\\n\
            } \\\n\
            for (i = offset_x; i < kernel_x_new; i = i + stride_w) \\\n\
            { \\\n\
                if ((x - i) < 0 || (x + (kernel_x_new - i)) > width_pad || i % dilation_w != 0) \\\n\
                { \\\n\
                    continue; \\\n\
                } \\\n\
                coord_in.x = (x - i + stride_w - 1) / stride_w + \\\n\
                             (y - j + stride_h - 1) / stride_h * move_time_x + \\\n\
                             (z - k + stride_d - 1) / stride_d * move_time_y * move_time_x; \\\n\
                coord_in.y = i / dilation_w + j * kernel_x / dilation_h + k * kernel_x * kernel_y / dilation_d; \\\n\
                sum = sum + convert_float(read_type(input, coord_in).x); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    sum = sum * inOutScale + inOutTile; \\\n\
    dst_type dst = 0; \\\n\
    dst.x = convert_type(sum); \\\n\
    write_type(output, coord_out, dst); \\\n\
}\n\
COL2IM(U32toU32, read_imageui, uint4,  convert_uint,  write_imageui)\n\
COL2IM(U32toI32, read_imageui, int4,   convert_int,   write_imagei)\n\
COL2IM(U32toF32, read_imageui, float4, convert_float, write_imagef)\n\
COL2IM(I32toU32, read_imagei,  uint4,  convert_uint,  write_imageui)\n\
COL2IM(I32toI32, read_imagei,  int4,   convert_int,   write_imagei)\n\
COL2IM(I32toF32, read_imagei,  float4, convert_float, write_imagef)\n\
COL2IM(F32toU32, read_imagef,  uint4,  convert_uint,  write_imageui)\n\
COL2IM(F32toI32, read_imagef,  int4,   convert_int,   write_imagei)\n\
COL2IM(F32toF32, read_imagef,  float4, convert_float, write_imagef)\n\
\n\
#define COL2IM_2D(name, read_type, dst_type ,convert_type, write_type) \\\n\
__kernel void col2im_##name##_2D \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride_w, \\\n\
                 int              stride_h, \\\n\
                 int              stride_d, \\\n\
                 int              dilation_w, \\\n\
                 int              dilation_h, \\\n\
                 int              dilation_d, \\\n\
                 int              pad_w_front, \\\n\
                 int              pad_w_end, \\\n\
                 int              pad_h_front, \\\n\
                 int              pad_h_end, \\\n\
                 int              pad_d_front, \\\n\
                 int              pad_d_end, \\\n\
                 int              kernel_x, \\\n\
                 int              kernel_y, \\\n\
                 int              kernel_z, \\\n\
                 float            inOutScale, \\\n\
                 float            inOutTile \\\n\
) \\\n\
{ \\\n\
    int x = get_global_id(0); \\\n\
    int y = get_global_id(1); \\\n\
    int z = get_global_id(2); \\\n\
    int4 coord_out = (int4)(x,y,z,0); \\\n\
    int4 coord_in = (int4)(0,0,z,0); \\\n\
 \\\n\
    float sum = 0.0f; \\\n\
    x = x + pad_w_front; \\\n\
    y = y + pad_h_front; \\\n\
    int offset_x = x % stride_w; \\\n\
    int offset_y = y % stride_h; \\\n\
    int i,j; \\\n\
    for (j = offset_y; j < kernel_y_new; j = j + stride_h) \\\n\
    { \\\n\
        if ((y - j) < 0 || (y + (kernel_y_new - j)) > height_pad || j % dilation_h != 0) \\\n\
        { \\\n\
            continue; \\\n\
        } \\\n\
        for (i = offset_x; i < kernel_x_new; i = i + stride_w) \\\n\
        { \\\n\
            if ((x - i) < 0 || (x + (kernel_x_new - i)) > width_pad || i % dilation_w != 0) \\\n\
            { \\\n\
                continue; \\\n\
            } \\\n\
            coord_in.x = (x - i + stride_w - 1) / stride_w + \\\n\
                         (y - j + stride_h - 1) / stride_h * move_time_x; \\\n\
            coord_in.y = i / dilation_w + j * kernel_x / dilation_h; \\\n\
            sum = sum + convert_float(read_type(input, coord_in).x); \\\n\
        } \\\n\
    } \\\n\
    sum = sum * inOutScale + inOutTile; \\\n\
    dst_type dst = 0; \\\n\
    dst.x = convert_type(sum); \\\n\
    write_type(output, coord_out, dst); \\\n\
}\n\
COL2IM_2D(U32toU32, read_imageui, uint4,  convert_uint,  write_imageui)\n\
COL2IM_2D(U32toI32, read_imageui, int4,   convert_int,   write_imagei)\n\
COL2IM_2D(U32toF32, read_imageui, float4, convert_float, write_imagef)\n\
COL2IM_2D(I32toU32, read_imagei,  uint4,  convert_uint,  write_imageui)\n\
COL2IM_2D(I32toI32, read_imagei,  int4,   convert_int,   write_imagei)\n\
COL2IM_2D(I32toF32, read_imagei,  float4, convert_float, write_imagef)\n\
COL2IM_2D(F32toU32, read_imagef,  uint4,  convert_uint,  write_imageui)\n\
COL2IM_2D(F32toI32, read_imagef,  int4,   convert_int,   write_imagei)\n\
COL2IM_2D(F32toF32, read_imagef,  float4, convert_float, write_imagef)"; /* end of col2im_cl*/

static const char crop_and_resize_bilinear_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
\n\
_viv_uniform float width_scale;\n\
_viv_uniform float height_scale;\n\
_viv_uniform int   image_width;\n\
_viv_uniform int   image_height;\n\
\n\
#define CROP_AND_RESIZE_BILINEAR(name, read_type, dst_type, conv_type, write_type) \\\n\
__kernel void crop_and_resize_bilinear_##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout, \\\n\
                 float            inOutScale, \\\n\
                 float            inOutTile, \\\n\
                 float            extrapolation_value \\\n\
) \\\n\
{ \\\n\
    int bb = get_global_id(2); \\\n\
    int y =  get_global_id(1); \\\n\
    int x = get_global_id(0); \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int2 coord_box_ind = (int2)(bb, 0); \\\n\
    int b = read_imagei(box_ind, coord_box_ind).x; \\\n\
    float4 xy; \\\n\
    float in_x, in_y; \\\n\
    int d = 0; \\\n\
 \\\n\
    Image img_boxes = create_image_from_image2d(boxes, 2); \\\n\
    __global half* boxes_ptr = (__global half*)img_boxes.ptr; \\\n\
    xy = vload_half4(bb, boxes_ptr); \\\n\
    float _width_scale = convert_float(xy.w - xy.y) * width_scale; \\\n\
    float _height_scale = convert_float(xy.z - xy.x) * height_scale; \\\n\
    if (_width_scale == 0) xy.y = 0.5 * (xy.y + xy.w); \\\n\
    if (_height_scale == 0) xy.x = 0.5 * (xy.x + xy.z); \\\n\
    in_y = xy.x * convert_float(image_height - 1) + convert_float(y) * _height_scale; \\\n\
    in_x = xy.y * convert_float(image_width - 1) + convert_float(x) * _width_scale; \\\n\
    float y_lerp = in_y - floor(in_y); \\\n\
    float x_lerp = in_x - floor(in_x); \\\n\
    float4 src0, src1, src2, src3; \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        int4 coord = (int4)(floor(in_x), floor(in_y), d + b * ori_depth, 0); \\\n\
        if (coord.x < 0 || coord.x > image_width - 1 || coord.y < 0 || coord.y > image_height - 1) \\\n\
        { \\\n\
            src0 = (float4)(extrapolation_value,0,0,0); \\\n\
        } \\\n\
        else \\\n\
        { \\\n\
            src0 = convert_float4(read_type(input, coord)); \\\n\
        } \\\n\
        coord.x = coord.x + 1; \\\n\
        if (coord.x < 0 || coord.x > image_width - 1 || coord.y < 0 || coord.y > image_height - 1) \\\n\
        { \\\n\
            src1 = (float4)(extrapolation_value,0,0,0); \\\n\
        } \\\n\
        else \\\n\
        { \\\n\
            src1 = convert_float4(read_type(input, coord)); \\\n\
        } \\\n\
        coord.y = coord.y + 1; \\\n\
        if (coord.x < 0 || coord.x > image_width - 1 || coord.y < 0 || coord.y > image_height - 1) \\\n\
        { \\\n\
            src3 = (float4)(extrapolation_value,0,0,0); \\\n\
        } \\\n\
        else \\\n\
        { \\\n\
            src3 = convert_float4(read_type(input, coord)); \\\n\
        } \\\n\
        coord.x = coord.x - 1; \\\n\
        if (coord.x < 0 || coord.x > image_width - 1 || coord.y < 0 || coord.y > image_height - 1) \\\n\
        { \\\n\
            src2 = (float4)(extrapolation_value,0,0,0); \\\n\
        } \\\n\
        else \\\n\
        { \\\n\
            src2 = convert_float4(read_type(input, coord)); \\\n\
        } \\\n\
        float4 top = src0 + (src1 - src0) * x_lerp; \\\n\
        float4 bottom = src2 + (src3 - src2) * x_lerp; \\\n\
        float4 value = top + (bottom - top) * y_lerp; \\\n\
        value = value * inOutScale + inOutTile; \\\n\
        dst_type dst = conv_type(value); \\\n\
        coord_out.z = d + coord_out.z * ori_depth; \\\n\
        write_type(output, coord_out, dst); \\\n\
    } \\\n\
}\n\
\n\
CROP_AND_RESIZE_BILINEAR(U32toU32,read_imageui, \\\n\
uint4, convert_uint4, write_imageui)\n\
CROP_AND_RESIZE_BILINEAR(U32toF32,read_imageui, \\\n\
float4,convert_float4,write_imagef)\n\
CROP_AND_RESIZE_BILINEAR(F32toF32,read_imagef, \\\n\
float4, convert_float4,write_imagef)\n\
CROP_AND_RESIZE_BILINEAR(F32toU32,read_imagef, \\\n\
uint4,  convert_uint4, write_imageui)\n\
CROP_AND_RESIZE_BILINEAR(F32toI32,read_imagef, \\\n\
int4,   convert_int4,  write_imagei)\n\
CROP_AND_RESIZE_BILINEAR(I32toI32,read_imagei,  \\\n\
int4,  convert_int4,  write_imagei)\n\
CROP_AND_RESIZE_BILINEAR(I32toF32,read_imagei,  \\\n\
float4,convert_float4,write_imagef)"; /* end of crop_and_resize_bilinear_cl*/

static const char crop_and_resize_nearest_neighbor_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
\n\
_viv_uniform float width_scale;\n\
_viv_uniform float height_scale;\n\
_viv_uniform int   image_width;\n\
_viv_uniform int   image_height;\n\
\n\
#define CROP_AND_RESIZE_NEAREST_NEIGHTBOR(name,src_type, read_type, dst_type, conv_type, write_type) \\\n\
__kernel void crop_and_resize_nearest_neighbor_##name \\\n\
( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_t         boxes, \\\n\
    __read_only image2d_t         box_ind, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 uint             ori_depth, \\\n\
                 uint             ori_batchout, \\\n\
                 float            inOutScale, \\\n\
                 float            inOutTile, \\\n\
                 float            extrapolation_value \\\n\
) \\\n\
{ \\\n\
    int bb = get_global_id(2); \\\n\
    int y =  get_global_id(1); \\\n\
    int x = get_global_id(0); \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int2 coord_box_ind = (int2)(bb, 0); \\\n\
    int b = read_imagei(box_ind, coord_box_ind).x; \\\n\
    float4 xy; \\\n\
    int in_x, in_y, d = 0; \\\n\
 \\\n\
    Image img_boxes = create_image_from_image2d(boxes, 2); \\\n\
    __global half* boxes_ptr = (__global half*)img_boxes.ptr; \\\n\
    xy = vload_half4(bb, boxes_ptr); \\\n\
    float _width_scale = convert_float(xy.w - xy.y) * width_scale; \\\n\
    float _height_scale = convert_float(xy.z - xy.x) * height_scale; \\\n\
    if (_width_scale == 0) xy.y = 0.5 * (xy.y + xy.w); \\\n\
    if (_height_scale == 0) xy.x = 0.5 * (xy.x + xy.z); \\\n\
    in_y = convert_int(round(xy.x * convert_float(image_height - 1) \\\n\
                                  + convert_float(y) * _height_scale)); \\\n\
    in_x = convert_int(round(xy.y * convert_float(image_width - 1) \\\n\
                                  + convert_float(x) * _width_scale)); \\\n\
    for (d = 0; d < ori_depth; d++) \\\n\
    { \\\n\
        int4 coord = (int4)(in_x, in_y, d + b * ori_depth, 0); \\\n\
        float4 src_f; \\\n\
        if (coord.x < 0 || coord.x > image_width - 1 || coord.y < 0 || coord.y > image_height - 1) \\\n\
        { \\\n\
            src_f = (float4)(extrapolation_value, 0, 0, 0); \\\n\
        } \\\n\
        else \\\n\
        { \\\n\
            src_type src = read_type(input, coord); \\\n\
            src_f = convert_float4(src); \\\n\
        } \\\n\
        src_f = src_f * inOutScale + inOutTile; \\\n\
        dst_type dst = conv_type(src_f); \\\n\
        coord_out.z = d + coord_out.z * ori_depth; \\\n\
        write_type(output, coord_out, dst); \\\n\
    } \\\n\
}\n\
\n\
CROP_AND_RESIZE_NEAREST_NEIGHTBOR(U32toU32,uint4, \\\n\
read_imageui, uint4, convert_uint4, write_imageui)\n\
CROP_AND_RESIZE_NEAREST_NEIGHTBOR(U32toF32,uint4, \\\n\
read_imageui, float4,convert_float4,write_imagef)\n\
CROP_AND_RESIZE_NEAREST_NEIGHTBOR(F32toF32,float4, \\\n\
read_imagef, float4,convert_float4,write_imagef)\n\
CROP_AND_RESIZE_NEAREST_NEIGHTBOR(F32toU32,float4, \\\n\
read_imagef, uint4, convert_uint4, write_imageui)\n\
CROP_AND_RESIZE_NEAREST_NEIGHTBOR(F32toI32,float4, \\\n\
read_imagef, int4,  convert_int4,  write_imagei)\n\
CROP_AND_RESIZE_NEAREST_NEIGHTBOR(I32toI32,int4,  \\\n\
read_imagei,  int4,  convert_int4,  write_imagei)\n\
CROP_AND_RESIZE_NEAREST_NEIGHTBOR(I32toF32,int4,  \\\n\
read_imagei,  float4,convert_float4,write_imagef)"; /* end of crop_and_resize_nearest_neighbor_cl*/

static const char cumsum_cl[] = "__kernel void cumsum_F32toF32_axis2(\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int channel,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    if(exclusive && rev)\n\
    {\n\
        coord_out.z = channel - 1;\n\
        write_imagef(output, coord_out, sum);\n\
\n\
        for(coord.z = channel - 1; coord.z > 0; coord.z--)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            coord_out.z--;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord_out, sum);\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord_out.z = 0;\n\
        write_imagef(output, coord_out, sum);\n\
        for(coord.z = 0; coord.z < channel - 1; coord.z++)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            coord_out.z++;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord_out, sum);\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord, sum);\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.z = 0; coord.z < channel; coord.z++)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord, sum);\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_toINT_AXIS2_SH(name, src_type, image_read, dst_type, image_write, convert_dtype) \\\n\
__kernel void cumsum_##name##_axis2( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int channel, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    dst_type dst = (dst_type)(0); \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst.x = convert_dtype(tmp_zp); \\\n\
 \\\n\
    float cnt = 0.0f; \\\n\
 \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord_out.z = channel - 1; \\\n\
        image_write(output, coord_out, dst); \\\n\
        for(coord.z = channel - 1; coord.z > 0; coord.z--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            coord_out.z--; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord_out, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        coord_out.z = 0; \\\n\
        image_write(output, coord_out, dst); \\\n\
        for(coord.z = 0; coord.z < channel - 1; coord.z++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            coord_out.z++; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord_out, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord, dst); \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord, dst); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_toINT_AXIS2_SH(U8toU8,   uint4,  read_imageui, uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_toINT_AXIS2_SH(F32toU8,  float4, read_imagef,  uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_toINT_AXIS2_SH(I32toI32, int4,   read_imagei,  int4,  write_imagei,  convert_int_sat_rte)\n\
\n\
__kernel void cumsum_F32toF32_axis1(\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int channel,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    if(exclusive && rev)\n\
    {\n\
        coord_out.y = height - 1;\n\
        write_imagef(output, coord_out, sum);\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            coord_out.y--;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord_out, sum);\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord_out.y = 0;\n\
        write_imagef(output, coord_out, sum);\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            coord_out.y++;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord_out, sum);\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord, sum);\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.y = 0; coord.y < height; coord.y++)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord, sum);\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_toINT_AXIS1_SH(name, src_type, image_read, dst_type, image_write, convert_dtype) \\\n\
__kernel void cumsum_##name##_axis1( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int channel, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    dst_type dst = (dst_type)(0); \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst.x = convert_dtype(tmp_zp); \\\n\
 \\\n\
    float cnt = 0; \\\n\
 \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord_out.y = height - 1; \\\n\
        image_write(output, coord_out, dst); \\\n\
 \\\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            coord_out.y--; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord_out, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        coord_out.y = 0; \\\n\
        image_write(output, coord_out, dst); \\\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            coord_out.y++; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord_out, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord, dst); \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height; coord.y++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord, dst); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_toINT_AXIS1_SH(U8toU8,   uint4,  read_imageui, uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_toINT_AXIS1_SH(F32toU8,  float4, read_imagef,  uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_toINT_AXIS1_SH(I32toI32, int4,   read_imagei,  int4,  write_imagei,  convert_int_sat_rte)\n\
\n\
__kernel void cumsum_F32toF32_axis0(\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int channel,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    if(exclusive && rev)\n\
    {\n\
        coord_out.x = width - 1;\n\
        write_imagef(output, coord_out, sum);\n\
        for(coord.x = width - 1; coord.x > 0; coord.x--)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            coord_out.x--;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord_out, sum);\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord_out.x = 0;\n\
        write_imagef(output, coord_out, sum);\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            coord_out.x++;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord_out, sum);\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord, sum);\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.x = 0; coord.x < width; coord.x++)\n\
        {\n\
            float4 data = read_imagef(input, coord);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord, sum);\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_toINT_AXIS0_SH(name, src_type, image_read, dst_type, image_write, convert_dtype) \\\n\
__kernel void cumsum_##name##_axis0( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int channel, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    dst_type dst = (dst_type)(0); \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst.x = convert_dtype(tmp_zp); \\\n\
 \\\n\
    float cnt = 0; \\\n\
 \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord_out.x = width - 1; \\\n\
        image_write(output, coord_out, dst); \\\n\
        for(coord.x = width - 1; coord.x > 0; coord.x--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            coord_out.x--; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord_out, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        coord_out.x = 0; \\\n\
        write_imageui(output, coord_out, dst); \\\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            coord_out.x++; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord_out, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord, dst); \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.x = 0; coord.x < width; coord.x++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord, dst); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_toINT_AXIS0_SH(U8toU8,   uint4,  read_imageui, uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_toINT_AXIS0_SH(F32toU8,  float4, read_imagef,  uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_toINT_AXIS0_SH(I32toI32, int4,   read_imagei,  int4,  write_imagei,  convert_int_sat_rte)\n\
\n\
"; /* end of cumsum_cl*/

static const char cumsum_2d_cl[] = "\n\
__kernel void cumsum_F32toF32_axis1_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    if(exclusive && rev)\n\
    {\n\
        coord.w = height - 1;\n\
        write_imagef(output, coord.zw, sum);\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            coord.w--;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.zw, sum);\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        write_imagef(output, coord.zw, sum);\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            coord.w++;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.zw, sum);\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.xy, sum);\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.y = 0; coord.y < height; coord.y++)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.xy, sum);\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_INT_AXIS1_2D_SH(name, src_type, image_read, dst_type, image_write, convert_dtype) \\\n\
__kernel void cumsum_##name##_axis1_2D( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int chn, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    dst_type dst = (dst_type)(0); \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst.x = convert_dtype(tmp_zp); \\\n\
 \\\n\
    float cnt = 0; \\\n\
 \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord.w = height - 1; \\\n\
        image_write(output, coord.zw, dst); \\\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            cnt += 1.0f; \\\n\
            coord.w--; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.zw, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        image_write(output, coord.zw, dst); \\\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            cnt += 1.0f; \\\n\
            coord.w++; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.zw, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.xy, dst); \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height; coord.y++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.xy, dst); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_INT_AXIS1_2D_SH(U8toU8,   uint4,  read_imageui, uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_INT_AXIS1_2D_SH(F32toU8,  float4, read_imagef,  uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_INT_AXIS1_2D_SH(I32toI32, int4,   read_imagei,  int4,  write_imagei,  convert_int_sat_rte)\n\
\n\
__kernel void cumsum_F32toF32_axis0_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    if(exclusive && rev)\n\
    {\n\
        coord.x = width - 1;\n\
        coord.z = coord.x;\n\
        write_imagef(output, coord.zw, sum);\n\
        for(; coord.x > 0; coord.x--)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            coord.z--;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.zw, sum);\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord.z = 0;\n\
        write_imagef(output, coord.zw, sum);\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            coord.z++;\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.zw, sum);\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.xy, sum);\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.x = 0; coord.x < width; coord.x++)\n\
        {\n\
            float4 data = read_imagef(input, coord.xy);\n\
            sum += data;\n\
\n\
            write_imagef(output, coord.xy, sum);\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_INT_AXIS0_2D_SH(name, src_type, image_read, dst_type, image_write, convert_dtype) \\\n\
__kernel void cumsum_##name##_axis0_2D( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int chn, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    dst_type dst = (dst_type)(0); \\\n\
 \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst.x = convert_dtype(tmp_zp); \\\n\
 \\\n\
    float cnt = 0.0f; \\\n\
 \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord.x = width - 1; \\\n\
        coord.z = coord.x; \\\n\
        image_write(output, coord.zw, dst); \\\n\
        for(; coord.x > 0; coord.x--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            coord.z--; \\\n\
            cnt += 1.0; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.zw, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        coord.z = 0; \\\n\
        image_write(output, coord.zw, dst); \\\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            cnt += 1.0f; \\\n\
            coord.z++; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.zw, dst); \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.xy, dst); \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.x = 0; coord.x < width; coord.x++) \\\n\
        { \\\n\
            src_type data = image_read(input, coord.xy); \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst.x = convert_dtype(tmpSum); \\\n\
            image_write(output, coord.xy, dst); \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_INT_AXIS0_2D_SH(U8toU8,   uint4,  read_imageui, uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_INT_AXIS0_2D_SH(F32toU8,  float4, read_imagef,  uint4, write_imageui, convert_uint_sat_rte)\n\
CUMSUM_INT_AXIS0_2D_SH(I32toI32, int4,   read_imagei,  int4,  write_imagei,  convert_int_sat_rte)\n\
"; /* end of cumsum_2d_cl*/

static const char cumsum_array_2d_axis0_cl[] = "\n\
__kernel void cumsum_array_F32toF32_axis0_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    float sum = (float)(0);\n\
    Image img1 = create_image_from_image2d(input, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global float* in_ptr = (__global float*)input_ptr;\n\
    __global float* out_ptr = (__global float*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord.x = width - 1;\n\
        coord.z = coord.x;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
\n\
        for(; coord.x > 0; coord.x--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord.z--;\n\
            sum += data;\n\
\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord.z = 0;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord.z++;\n\
            sum += data;\n\
\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.x = 0; coord.x < width; coord.x++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void cumsum_array_U8toU8_axis0_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    uint sum = (uint)(0);\n\
    uint dst = (uint)(0);\n\
\n\
    int tmp_zp = convert_int_rte(output_zp);\n\
    dst.x = convert_uint_sat(tmp_zp);\n\
\n\
    float cnt = 0.0f;\n\
\n\
    Image img1 = create_image_from_image2d(input, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global uint* in_ptr = (__global uint*)input_ptr;\n\
    __global uint* out_ptr = (__global uint*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord.x = width - 1;\n\
        coord.z = coord.x;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(; coord.x > 0; coord.x--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            coord.z--;\n\
            cnt += 1.0;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord.z = 0;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            coord.z++;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.x = 0; coord.x < width; coord.x++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void cumsum_array_F32toU8_axis0_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    float4 sum = (float4)(0);\n\
    uint4 dst = (uint4)(0);\n\
    int tmp_zp = convert_int_rte(output_zp);\n\
    dst.x = convert_uint_sat(tmp_zp);\n\
\n\
    float cnt = 0.0f;\n\
    Image img1 = create_image_from_image2d(input, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global float* in_ptr = (__global float*)input_ptr;\n\
    __global uint* out_ptr = (__global uint*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord.x = width - 1;\n\
        coord.z = coord.x;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global uint*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(; coord.x > 0; coord.x--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord.z--;\n\
            cnt += 1.0;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord.z = 0;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global uint*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            coord.z++;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.x = 0; coord.x < width; coord.x++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum.x * in_out_scale + tmpAlpha;\n\
\n\
            dst.x = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
"; /* end of cumsum_array_2d_axis0_cl*/

static const char cumsum_array_2d_axis1_cl[] = "\n\
__kernel void cumsum_array_F32toF32_axis1_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    float sum = (float)(0);\n\
    Image img1 = create_image_from_image2d(input, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global float* in_ptr = (__global float*)input_ptr;\n\
    __global float* out_ptr = (__global float*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord.w = height - 1;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord.w--;\n\
            sum += data;\n\
\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        write_imagef(output, coord.zw, sum);\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord.w++;\n\
            sum += data;\n\
\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.y = 0; coord.y < height; coord.y++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void cumsum_array_U8toU8_axis1_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    uint sum = (uint)(0);\n\
    uint dst = (uint)(0);\n\
\n\
    int tmp_zp = convert_int_rte(output_zp);\n\
    dst = convert_uint_sat(tmp_zp);\n\
\n\
    float cnt = 0;\n\
    Image img1 = create_image_from_image2d(input, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global uint* in_ptr = (__global uint*)input_ptr;\n\
    __global uint* out_ptr = (__global uint*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord.w = height - 1;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            coord.w--;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            coord.w++;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.y = 0; coord.y < height; coord.y++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global uint*)input_ptr;\n\
            uint data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void cumsum_array_F32toU8_axis1_2D(\n\
    __read_only image2d_t  input,\n\
    __write_only image2d_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    float sum = (float)(0);\n\
    uint dst = (uint)(0);\n\
    int tmp_zp = convert_int_rte(output_zp);\n\
    dst = convert_uint_sat(tmp_zp);\n\
\n\
    float cnt = 0;\n\
    Image img1 = create_image_from_image2d(input, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global float* in_ptr = (__global float*)input_ptr;\n\
    __global uint* out_ptr = (__global uint*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord.w = height - 1;\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global uint*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            coord.w--;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
        out_ptr = (__global uint*)output_ptr;\n\
        out_ptr[0] = dst;\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            coord.w++;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.zw);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.y = 0; coord.y < height; coord.y++)\n\
        {\n\
            input_ptr = get_image_ptr_from_coord(img1, coord.xy);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            cnt += 1.0f;\n\
            sum += data;\n\
\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp;\n\
            float tmpSum = sum * in_out_scale + tmpAlpha;\n\
\n\
            dst = (uint)convert_int_rte(tmpSum);\n\
            output_ptr = get_image_ptr_from_coord(img2, coord.xy);\n\
            out_ptr = (__global uint*)output_ptr;\n\
            out_ptr[0] = dst;\n\
        }\n\
    }\n\
}\n\
"; /* end of cumsum_array_2d_axis1_cl*/

static const char cumsum_array_axis0_cl[] = "\n\
__kernel void cumsum_array_F32toF32_axis0(\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int channel,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    float sum = (float)(0);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 4);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 4);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global float* in_ptr = (__global float*)input_ptr;\n\
    __global float* out_ptr = (__global float*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord_out.x = width - 1;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
        for(coord.x = width - 1; coord.x > 0; coord.x--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord_out.x--;\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord_out.x = 0;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord_out.x++;\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.x = 0; coord.x < width; coord.x++)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_ARRAY_toU8_AXIS0_SH(name, src_type) \\\n\
__kernel void cumsum_array_##name##toU8_axis0( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int channel, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    uint dst = (uint)(0); \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst = convert_uint_sat(tmp_zp); \\\n\
 \\\n\
    float cnt = 0; \\\n\
 \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 4); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 4); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
    __global uint* out_ptr = (__global uint*)output_ptr; \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord_out.x = width - 1; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
        out_ptr = (__global uint*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
        for(coord.x = width - 1; coord.x > 0; coord.x--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            coord_out.x--; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        coord_out.x = 0; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
        out_ptr = (__global uint*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
        for(coord.x = 0; coord.x < width - 1; coord.x++) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            coord_out.x++; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.x = width - 1; coord.x >= 0; coord.x--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.x = 0; coord.x < width; coord.x++) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_toU8_AXIS0_SH(U8,uint)\n\
CUMSUM_ARRAY_toU8_AXIS0_SH(F32,float)\n\
"; /* end of cumsum_array_axis0_cl*/

static const char cumsum_array_axis1_cl[] = "\n\
__kernel void cumsum_array_F32toF32_axis1(\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int channel,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    float sum = (float)(0);\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 4);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 4);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global float* in_ptr = (__global float*)input_ptr;\n\
    __global float* out_ptr = (__global float*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord_out.y = height - 1;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord_out.y--;\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord_out.y = 0;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord_out.y++;\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.y = 0; coord.y < height; coord.y++)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_ARRAY_toU8_AXIS1_SH(name, src_type) \\\n\
__kernel void cumsum_array_##name##toU8_axis1( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int channel, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    uint dst = (uint4)(0); \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst = convert_uint_sat(tmp_zp); \\\n\
 \\\n\
    float cnt = 0; \\\n\
 \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 4); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 4); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
    __global uint* out_ptr = (__global uint*)output_ptr; \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord_out.y = height - 1; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
        out_ptr = (__global uint*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
 \\\n\
        for(coord.y = height - 1; coord.y > 0; coord.y--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            coord_out.y--; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        coord_out.y = 0; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
        out_ptr = (__global uint*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
        for(coord.y = 0; coord.y < height - 1; coord.y++) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            coord_out.y++; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.y = height - 1; coord.y >= 0; coord.y--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height; coord.y++) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_toU8_AXIS1_SH(U8,uint)\n\
CUMSUM_ARRAY_toU8_AXIS1_SH(F32,float)\n\
"; /* end of cumsum_array_axis1_cl*/

static const char cumsum_array_axis2_cl[] = "__kernel void cumsum_array_F32toF32_axis2(\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
    int axis,\n\
    int exclusive,\n\
    int rev,\n\
    int width,\n\
    int height,\n\
    int channel,\n\
    int input_zp,\n\
    float in_out_scale,\n\
    float in_out_zp_scale,\n\
    float output_zp\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    float sum = 0;\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 4);\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 4);\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
    __global float* in_ptr = (__global float*)input_ptr;\n\
    __global float* out_ptr = (__global float*)output_ptr;\n\
    if(exclusive && rev)\n\
    {\n\
        coord_out.z = channel - 1;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
\n\
        for(coord.z = channel - 1; coord.z > 0; coord.z--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord_out.z--;\n\
            sum += data;\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(exclusive)\n\
    {\n\
        coord_out.z = 0;\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
        out_ptr = (__global float*)output_ptr;\n\
        out_ptr[0] = sum;\n\
        for(coord.z = 0; coord.z < channel - 1; coord.z++)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            coord_out.z++;\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else if(rev)\n\
    {\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
    else\n\
    {\n\
        for(coord.z = 0; coord.z < channel; coord.z++)\n\
        {\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord);\n\
            in_ptr = (__global float*)input_ptr;\n\
            float data = in_ptr[0];\n\
            sum += data;\n\
\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord);\n\
            out_ptr = (__global float*)output_ptr;\n\
            out_ptr[0] = sum;\n\
        }\n\
    }\n\
}\n\
\n\
#define CUMSUM_ARRAY_toU8_AXIS2_SH(name, src_type) \\\n\
__kernel void cumsum_array_##name##toU8_axis2( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int axis, \\\n\
    int exclusive, \\\n\
    int rev, \\\n\
    int width, \\\n\
    int height, \\\n\
    int channel, \\\n\
    int input_zp, \\\n\
    float in_out_scale, \\\n\
    float in_out_zp_scale, \\\n\
    float output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out = coord; \\\n\
 \\\n\
    src_type sum = (src_type)(0); \\\n\
    uint dst = (uint)(0); \\\n\
    int tmp_zp = convert_int_rte(output_zp); \\\n\
    dst = convert_uint_sat(tmp_zp); \\\n\
 \\\n\
    float cnt = 0.0f; \\\n\
    Tensor img1 = create_tensor_from_image2d_array(input, 4); \\\n\
    Tensor img2 = create_tensor_from_image2d_array(output, 4); \\\n\
    uchar* input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
    uchar* output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
    __global src_type* in_ptr = (__global src_type*)input_ptr; \\\n\
    __global uint* out_ptr = (__global uint*)output_ptr; \\\n\
 \\\n\
    if(exclusive && rev) \\\n\
    { \\\n\
        coord_out.z = channel - 1; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
        out_ptr = (__global uint*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
        for(coord.z = channel - 1; coord.z > 0; coord.z--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            coord_out.z--; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(exclusive) \\\n\
    { \\\n\
        coord_out.z = 0; \\\n\
        output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
        out_ptr = (__global uint*)output_ptr; \\\n\
        out_ptr[0] = dst; \\\n\
        for(coord.z = 0; coord.z < channel - 1; coord.z++) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            coord_out.z++; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord_out); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else if(rev) \\\n\
    { \\\n\
        for(coord.z = channel - 1; coord.z >= 0; coord.z--) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
        { \\\n\
            input_ptr = get_tensor_ptr_from_coord(img1, coord); \\\n\
            in_ptr = (__global src_type*)input_ptr; \\\n\
            src_type data = in_ptr[0]; \\\n\
            cnt += 1.0f; \\\n\
            sum += data; \\\n\
 \\\n\
            float tmpAlpha = cnt * in_out_zp_scale + output_zp; \\\n\
            float tmpSum = sum * in_out_scale + tmpAlpha; \\\n\
 \\\n\
            dst = (uint)convert_int_rte(tmpSum); \\\n\
            output_ptr = get_tensor_ptr_from_coord(img2, coord); \\\n\
            out_ptr = (__global uint*)output_ptr; \\\n\
            out_ptr[0] = dst; \\\n\
        } \\\n\
    } \\\n\
}\n\
CUMSUM_ARRAY_toU8_AXIS2_SH(U8,uint)\n\
CUMSUM_ARRAY_toU8_AXIS2_SH(F32,float)\n\
\n\
"; /* end of cumsum_array_axis2_cl*/

static const char depth2space_crd_cl[] = "\n\
__kernel void depth2space_crd_F32toF32(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int block_e2 = block_size * block_size;\n\
    ushort blk = (ushort)block_size;\n\
    int inx = (int)((ushort)gidx / blk);\n\
    int iny = (int)((ushort)gidy / blk);\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2;\n\
    int4 coord_in = (int4)(inx, iny, inz, 0);\n\
    float4 data = read_imagef(input, coord_in);\n\
    write_imagef(output, coord_out, data);\n\
}\n\
"; /* end of depth2space_crd_cl*/

static const char eltwise_ops_helper_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
typedef struct Image\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
} Image;\n\
\n\
inline uchar* get_image_ptr_from_coord(Image img, int2 coord)\n\
{\n\
    return img.ptr + coord.x * img.stride_x + coord.y * img.stride_y;\n\
}\n\
\n\
inline Image create_image_from_image2d(image2d_t input, int stride_x)\n\
{\n\
    int stride_y;\n\
#if (USE_40BITS_VA==0)\n\
    int8 desc;\n\
#else\n\
    int8 desc;\n\
    _viv_asm(GET_IMAGE_STRIDE, stride_y, input);\n\
#endif\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
    uint address = as_uint(desc.s0);\n\
\n\
#if (USE_40BITS_VA==0)\n\
    stride_y = desc.s1;\n\
#endif\n\
\n\
    Image img =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = stride_y\n\
    };\n\
\n\
    return img;\n\
}\n\
\n\
typedef struct Tensor\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
    int             stride_z;\n\
} Tensor;\n\
\n\
inline uchar* get_tensor_ptr_from_coord(Tensor t, int4 coord)\n\
{\n\
    return t.ptr + coord.x * t.stride_x + coord.y * t.stride_y + coord.z * t.stride_z;\n\
}\n\
\n\
inline Tensor create_tensor_from_image2d_array(image2d_array_t input, int stride_x)\n\
{\n\
    int8 desc;\n\
    int2 strides;\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
\n\
#if (USE_40BITS_VA==0)\n\
    strides.x = desc.s1;\n\
    strides.y = desc.s4;\n\
#else\n\
    _viv_asm(GET_IMAGE_STRIDE, strides, input);\n\
#endif\n\
    uint address = as_uint(desc.s0);\n\
\n\
    Tensor t =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = strides.x,\n\
        .stride_z                      = strides.y\n\
    };\n\
\n\
    return t;\n\
}\n\
\n\
#define READ_IMAGEF_2DARRAY(dest, tensor, coord) \\\n\
    do { \\\n\
        int depth = get_image_array_size(tensor); \\\n\
        int4 coord_in = coord; \\\n\
        _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
        dest = read_imagef(tensor, coord_in); \\\n\
       } while(0)\n\
\n\
#define READ_IMAGEI_2DARRAY(dest, tensor, coord) \\\n\
    do { \\\n\
        int depth = get_image_array_size(tensor); \\\n\
        int4 coord_in = coord; \\\n\
        _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
        dest = read_imagei(tensor, coord_in); \\\n\
       } while(0)\n\
\n\
#define READ_IMAGEUI_2DARRAY(dest, tensor, coord) \\\n\
    do { \\\n\
        int depth = get_image_array_size(tensor); \\\n\
        int4 coord_in = coord; \\\n\
        _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
        dest = read_imageui(tensor, coord_in); \\\n\
       } while(0)\n\
"; /* end of eltwise_ops_helper_cl*/

static const char eltwise_unary_0_cl[] = "float eltwise_unary_sin(float x, float alpha, float beta)\n\
{\n\
    return native_sin(x);\n\
}\n\
\n\
float eltwise_unary_cos(float x, float alpha, float beta)\n\
{\n\
    return native_cos(x);\n\
}\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
float eltwise_unary_exp(float x, float alpha, float beta)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
#define rlogE    (0.693147182f)\n\
float eltwise_unary_log(float x, float alpha, float beta)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float eltwise_unary_neg(float x, float alpha, float beta)\n\
{\n\
    return x * -1;\n\
}\n\
\n\
float eltwise_unary_hard_sigmoid(float x, float alpha, float beta)\n\
{\n\
    x = alpha * x + beta;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
float _softrelu(float x, float alpha)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    x += 1;\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float _tanh(float x, float alpha)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return (2 * x - 1);\n\
}\n\
\n\
float eltwise_unary_mish(float x, float alpha, float beta)\n\
{\n\
    float y = _softrelu(x, alpha);\n\
    x = x * _tanh(y, alpha);\n\
    return x;\n\
}\n\
\n\
float eltwise_unary_round(float x, float alpha, float beta)\n\
{\n\
    return convert_float(convert_int_rte(x));\n\
}\n\
\n\
float evaluate_polynomial_alpha(float x2)\n\
{\n\
    float4 alpha0 = (float4){-2.72614225801306e-10f, 2.77068142495902e-08f,\n\
                            -2.10102402082508e-06f, -5.69250639462346e-05f};\n\
    float4 alpha1 = (float4){-7.34990630326855e-04f, -2.95459980854025e-03f,\n\
                            -1.60960333262415e-02f, 0};\n\
\n\
    float poly = alpha0.x * x2 + alpha0.y;\n\
    poly = poly * x2 + alpha0.z;\n\
    poly = poly * x2 + alpha0.w;\n\
    poly = poly * x2 + alpha1.x;\n\
    poly = poly * x2 + alpha1.y;\n\
    poly = poly * x2 + alpha1.z;\n\
\n\
    return poly;\n\
}\n\
\n\
float evaluate_polynomial_beta(float x2)\n\
{\n\
    float4 beta0 = (float4){-1.45660718464996e-05f, -2.13374055278905e-04f,\n\
                            -1.68282697438203e-03f, -7.37332916720468e-03f};\n\
    float4 beta1 = (float4){-1.42647390514189e-02f, 0, 0, 0};\n\
\n\
    float poly = beta0.x * x2 + beta0.y;\n\
    poly = poly * x2 + beta0.z;\n\
    poly = poly * x2 + beta0.w;\n\
    poly = poly * x2 + beta1.x;\n\
\n\
    return 1.0f / poly;\n\
}\n\
\n\
float erf_eval(float _x)\n\
{\n\
    float x = clamp(_x, -4, 4);\n\
    float x2 = x * x;\n\
\n\
    return x * evaluate_polynomial_alpha(x2) * evaluate_polynomial_beta(x2);\n\
}\n\
\n\
#define RSQRT2      (0.70710678118654752440084436210485f)\n\
float eltwise_unary_gelu(float x, float alpha, float beta)\n\
{\n\
    x = 0.5f * x * (1 + erf_eval(x * RSQRT2));\n\
\n\
    return x;\n\
}\n\
\n\
#define SQRT_2_RCP_PI  0.7978845834732056f\n\
float eltwise_unary_hard_gelu(float x, float alpha, float beta)\n\
{\n\
    float cdf = 0.5f + 0.5f * _tanh(SQRT_2_RCP_PI *\n\
                        (x + 0.044715f * x * x * x), 0);\n\
    return x * cdf;\n\
}\n\
\n\
float eltwise_unary_selu(float val, float alpha_times_gamma, float gamma)\n\
{\n\
    float x = val * logE;\n\
    x = exp2(x) * alpha_times_gamma - alpha_times_gamma;\n\
\n\
    return val <= 0 ? x : val * gamma;\n\
}\n\
\n\
float eltwise_unary_celu(float val, float alpha, float rcp_alpha)\n\
{\n\
    float x = val * logE * rcp_alpha;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val;\n\
}\n\
\n\
float eltwise_unary_rcp(float val, float alpha, float rcp_alpha)\n\
{\n\
    return 1.0f / val;\n\
}\n\
\n\
float eltwise_unary_sign(float val, float alpha, float rcp_alpha)\n\
{\n\
    return sign(val);\n\
}\n\
\n\
float eltwise_unary_softsign(float val, float alpha, float rcp_alpha)\n\
{\n\
    return val / (1.0f + fabs(val));\n\
}\n\
\n\
float eltwise_unary_atan(float x, float alpha, float beta)\n\
{\n\
    return atan(x);\n\
}\n\
\n\
float eltwise_unary_atanh(float x, float alpha, float beta)\n\
{\n\
    return atanh(x);\n\
}\n\
\n\
float eltwise_unary_acosh(float x, float alpha, float beta)\n\
{\n\
    return acosh(x);\n\
}\n\
\n\
float eltwise_unary_inverse_sigmoid(float x, float alpha, float beta)\n\
{\n\
    float x1, x2;\n\
    x = clamp(x, 0, 1);\n\
    x1 = x > alpha ? x : alpha;\n\
    x2 = 1 - x;\n\
    x2 = x2 > alpha ? x2 : alpha;\n\
    return log(x1 / x2);\n\
}\n\
\n\
float eltwise_unary_tan(float x, float alpha, float beta)\n\
{\n\
    return native_tan(x);\n\
}\n\
\n\
\n\
#define ELTWISE_UNARY_F32_2D(func_name) \\\n\
__kernel void func_name##_F32toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP, \\\n\
                 float     alpha, \\\n\
                 float           beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x, alpha, beta); \\\n\
 \\\n\
    write_imagef(output, coord, dst.xxxx); \\\n\
}\n\
ELTWISE_UNARY_F32_2D(sin)\n\
ELTWISE_UNARY_F32_2D(cos)\n\
ELTWISE_UNARY_F32_2D(exp)\n\
ELTWISE_UNARY_F32_2D(log)\n\
ELTWISE_UNARY_F32_2D(neg)\n\
ELTWISE_UNARY_F32_2D(mish)\n\
ELTWISE_UNARY_F32_2D(hard_sigmoid)\n\
ELTWISE_UNARY_F32_2D(round)\n\
ELTWISE_UNARY_F32_2D(gelu)\n\
ELTWISE_UNARY_F32_2D(hard_gelu)\n\
ELTWISE_UNARY_F32_2D(selu)\n\
ELTWISE_UNARY_F32_2D(celu)\n\
ELTWISE_UNARY_F32_2D(rcp)\n\
ELTWISE_UNARY_F32_2D(sign)\n\
ELTWISE_UNARY_F32_2D(softsign)\n\
ELTWISE_UNARY_F32_2D(atan)\n\
ELTWISE_UNARY_F32_2D(atanh)\n\
ELTWISE_UNARY_F32_2D(acosh)\n\
ELTWISE_UNARY_F32_2D(inverse_sigmoid)\n\
ELTWISE_UNARY_F32_2D(tan)\n\
\n\
#define ELTWISE_UNARY_U8_2D(func_name) \\\n\
__kernel void func_name##_U8toU8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP, \\\n\
                 float     alpha, \\\n\
                 float     beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x, alpha, beta); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8_2D(sin)\n\
ELTWISE_UNARY_U8_2D(cos)\n\
ELTWISE_UNARY_U8_2D(exp)\n\
ELTWISE_UNARY_U8_2D(log)\n\
ELTWISE_UNARY_U8_2D(neg)\n\
ELTWISE_UNARY_U8_2D(mish)\n\
ELTWISE_UNARY_U8_2D(hard_sigmoid)\n\
ELTWISE_UNARY_U8_2D(round)\n\
ELTWISE_UNARY_U8_2D(gelu)\n\
ELTWISE_UNARY_U8_2D(hard_gelu)\n\
ELTWISE_UNARY_U8_2D(selu)\n\
ELTWISE_UNARY_U8_2D(celu)\n\
ELTWISE_UNARY_U8_2D(rcp)\n\
ELTWISE_UNARY_U8_2D(sign)\n\
ELTWISE_UNARY_U8_2D(softsign)\n\
ELTWISE_UNARY_U8_2D(atan)\n\
ELTWISE_UNARY_U8_2D(atanh)\n\
ELTWISE_UNARY_U8_2D(acosh)\n\
ELTWISE_UNARY_U8_2D(inverse_sigmoid)\n\
ELTWISE_UNARY_U8_2D(tan)\n\
\n\
#define ELTWISE_UNARY_U8toF32_2D(func_name) \\\n\
__kernel void func_name##_U8toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP, \\\n\
                 float     alpha, \\\n\
                 float     beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 dst = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    dst.x = eltwise_unary_##func_name(dst.x, alpha, beta); \\\n\
 \\\n\
    write_imagef(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8toF32_2D(sin)\n\
ELTWISE_UNARY_U8toF32_2D(cos)\n\
ELTWISE_UNARY_U8toF32_2D(exp)\n\
ELTWISE_UNARY_U8toF32_2D(log)\n\
ELTWISE_UNARY_U8toF32_2D(neg)\n\
ELTWISE_UNARY_U8toF32_2D(mish)\n\
ELTWISE_UNARY_U8toF32_2D(hard_sigmoid)\n\
ELTWISE_UNARY_U8toF32_2D(round)\n\
ELTWISE_UNARY_U8toF32_2D(gelu)\n\
ELTWISE_UNARY_U8toF32_2D(hard_gelu)\n\
ELTWISE_UNARY_U8toF32_2D(selu)\n\
ELTWISE_UNARY_U8toF32_2D(celu)\n\
ELTWISE_UNARY_U8toF32_2D(rcp)\n\
ELTWISE_UNARY_U8toF32_2D(sign)\n\
ELTWISE_UNARY_U8toF32_2D(softsign)\n\
ELTWISE_UNARY_U8toF32_2D(atan)\n\
ELTWISE_UNARY_U8toF32_2D(atanh)\n\
ELTWISE_UNARY_U8toF32_2D(acosh)\n\
ELTWISE_UNARY_U8toF32_2D(inverse_sigmoid)\n\
ELTWISE_UNARY_U8toF32_2D(tan)\n\
\n\
__kernel void neg_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail,\n\
                 float     outputScale,\n\
                 float     outputZP,\n\
                 float     alpha,\n\
                 float     beta\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src = read_imagei(input, coord);\n\
\n\
    int4 dst = -src;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of eltwise_unary_0_cl*/

static const char eltwise_unary_1_cl[] = "float eltwise_unary_sin(float x, float alpha, float beta)\n\
{\n\
    return native_sin(x);\n\
}\n\
\n\
float eltwise_unary_cos(float x, float alpha, float beta)\n\
{\n\
    return native_cos(x);\n\
}\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
float eltwise_unary_exp(float x, float alpha, float beta)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
#define rlogE    (0.693147182f)\n\
float eltwise_unary_log(float x, float alpha, float beta)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float eltwise_unary_neg(float x, float alpha, float beta)\n\
{\n\
    return x * -1;\n\
}\n\
\n\
float eltwise_unary_hard_sigmoid(float x, float alpha, float beta)\n\
{\n\
    x = alpha * x + beta;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
float _softrelu(float x, float alpha)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    x += 1;\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float _tanh(float x, float alpha)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return (2 * x - 1);\n\
}\n\
\n\
float eltwise_unary_mish(float x, float alpha, float beta)\n\
{\n\
    float y = _softrelu(x, alpha);\n\
    x = x * _tanh(y, alpha);\n\
    return x;\n\
}\n\
\n\
float eltwise_unary_round(float x, float alpha, float beta)\n\
{\n\
    return convert_float(convert_int_rte(x));\n\
}\n\
\n\
float evaluate_polynomial_alpha(float x2)\n\
{\n\
    float4 alpha0 = (float4){-2.72614225801306e-10f, 2.77068142495902e-08f,\n\
                            -2.10102402082508e-06f, -5.69250639462346e-05f};\n\
    float4 alpha1 = (float4){-7.34990630326855e-04f, -2.95459980854025e-03f,\n\
                            -1.60960333262415e-02f, 0};\n\
\n\
    float poly = alpha0.x * x2 + alpha0.y;\n\
    poly = poly * x2 + alpha0.z;\n\
    poly = poly * x2 + alpha0.w;\n\
    poly = poly * x2 + alpha1.x;\n\
    poly = poly * x2 + alpha1.y;\n\
    poly = poly * x2 + alpha1.z;\n\
\n\
    return poly;\n\
}\n\
\n\
float evaluate_polynomial_beta(float x2)\n\
{\n\
    float4 beta0 = (float4){-1.45660718464996e-05f, -2.13374055278905e-04f,\n\
                            -1.68282697438203e-03f, -7.37332916720468e-03f};\n\
    float4 beta1 = (float4){-1.42647390514189e-02f, 0, 0, 0};\n\
\n\
    float poly = beta0.x * x2 + beta0.y;\n\
    poly = poly * x2 + beta0.z;\n\
    poly = poly * x2 + beta0.w;\n\
    poly = poly * x2 + beta1.x;\n\
\n\
    return 1.0f / poly;\n\
}\n\
\n\
float erf_eval(float _x)\n\
{\n\
    float x = clamp(_x, -4, 4);\n\
    float x2 = x * x;\n\
\n\
    return x * evaluate_polynomial_alpha(x2) * evaluate_polynomial_beta(x2);\n\
}\n\
\n\
#define RSQRT2      (0.70710678118654752440084436210485f)\n\
float eltwise_unary_gelu(float x, float alpha, float beta)\n\
{\n\
    x = 0.5f * x * (1 + erf_eval(x * RSQRT2));\n\
\n\
    return x;\n\
}\n\
\n\
#define SQRT_2_RCP_PI  0.7978845834732056f\n\
float eltwise_unary_hard_gelu(float x, float alpha, float beta)\n\
{\n\
    float cdf = 0.5f + 0.5f * _tanh(SQRT_2_RCP_PI *\n\
                        (x + 0.044715f * x * x * x), 0);\n\
    return x * cdf;\n\
}\n\
\n\
float eltwise_unary_selu(float val, float alpha_times_gamma, float gamma)\n\
{\n\
    float x = val * logE;\n\
    x = exp2(x) * alpha_times_gamma - alpha_times_gamma;\n\
\n\
    return val < 0 ? x : val * gamma;\n\
}\n\
\n\
float eltwise_unary_celu(float val, float alpha, float rcp_alpha)\n\
{\n\
    float x = val * logE * rcp_alpha;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val;\n\
}\n\
\n\
float eltwise_unary_rcp(float val, float alpha, float rcp_alpha)\n\
{\n\
    return 1.0f / val;\n\
}\n\
\n\
float eltwise_unary_sign(float val, float alpha, float rcp_alpha)\n\
{\n\
    return sign(val);\n\
}\n\
\n\
float eltwise_unary_softsign(float val, float alpha, float rcp_alpha)\n\
{\n\
    return val / (1.0f + fabs(val));\n\
}\n\
\n\
float eltwise_unary_atan(float x, float alpha, float beta)\n\
{\n\
    return atan(x);\n\
}\n\
\n\
float eltwise_unary_atanh(float x, float alpha, float beta)\n\
{\n\
    return atanh(x);\n\
}\n\
float eltwise_unary_acosh(float x, float alpha, float beta)\n\
{\n\
    return acosh(x);\n\
}\n\
\n\
float eltwise_unary_inverse_sigmoid(float x, float alpha, float beta)\n\
{\n\
    float x1, x2;\n\
    x = clamp(x, 0, 1);\n\
    x1 = x > alpha ? x : alpha;\n\
    x2 = 1 - x;\n\
    x2 = x2 > alpha ? x2 : alpha;\n\
    return log(x1 / x2);\n\
}\n\
\n\
float eltwise_unary_tan(float x, float alpha, float beta)\n\
{\n\
    return native_tan(x);\n\
}\n\
\n\
#define ELTWISE_UNARY_F32(func_name) \\\n\
__kernel void func_name##_F32toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP, \\\n\
                 float           alpha, \\\n\
                 float           beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x, alpha, beta); \\\n\
 \\\n\
    write_imagef(output, coord, dst.xxxx); \\\n\
}\n\
ELTWISE_UNARY_F32(sin)\n\
ELTWISE_UNARY_F32(cos)\n\
ELTWISE_UNARY_F32(exp)\n\
ELTWISE_UNARY_F32(log)\n\
ELTWISE_UNARY_F32(neg)\n\
ELTWISE_UNARY_F32(mish)\n\
ELTWISE_UNARY_F32(hard_sigmoid)\n\
ELTWISE_UNARY_F32(round)\n\
ELTWISE_UNARY_F32(gelu)\n\
ELTWISE_UNARY_F32(hard_gelu)\n\
ELTWISE_UNARY_F32(selu)\n\
ELTWISE_UNARY_F32(celu)\n\
ELTWISE_UNARY_F32(rcp)\n\
ELTWISE_UNARY_F32(sign)\n\
ELTWISE_UNARY_F32(softsign)\n\
ELTWISE_UNARY_F32(atan)\n\
ELTWISE_UNARY_F32(atanh)\n\
ELTWISE_UNARY_F32(acosh)\n\
ELTWISE_UNARY_F32(inverse_sigmoid)\n\
ELTWISE_UNARY_F32(tan)\n\
\n\
#define ELTWISE_UNARY_U8(func_name) \\\n\
__kernel void func_name##_U8toU8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP, \\\n\
                 float           alpha, \\\n\
                 float           beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x, alpha, beta); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8(sin)\n\
ELTWISE_UNARY_U8(cos)\n\
ELTWISE_UNARY_U8(exp)\n\
ELTWISE_UNARY_U8(log)\n\
ELTWISE_UNARY_U8(neg)\n\
ELTWISE_UNARY_U8(mish)\n\
ELTWISE_UNARY_U8(hard_sigmoid)\n\
ELTWISE_UNARY_U8(round)\n\
ELTWISE_UNARY_U8(gelu)\n\
ELTWISE_UNARY_U8(hard_gelu)\n\
ELTWISE_UNARY_U8(selu)\n\
ELTWISE_UNARY_U8(celu)\n\
ELTWISE_UNARY_U8(rcp)\n\
ELTWISE_UNARY_U8(sign)\n\
ELTWISE_UNARY_U8(softsign)\n\
ELTWISE_UNARY_U8(atan)\n\
ELTWISE_UNARY_U8(atanh)\n\
ELTWISE_UNARY_U8(acosh)\n\
ELTWISE_UNARY_U8(inverse_sigmoid)\n\
ELTWISE_UNARY_U8(tan)\n\
\n\
#define ELTWISE_UNARY_U8toF32(func_name) \\\n\
__kernel void func_name##_U8toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP, \\\n\
                 float           alpha, \\\n\
                 float           beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 dst = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    dst.x = eltwise_unary_##func_name(dst.x, alpha, beta); \\\n\
 \\\n\
    write_imagef(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8toF32(sin)\n\
ELTWISE_UNARY_U8toF32(cos)\n\
ELTWISE_UNARY_U8toF32(exp)\n\
ELTWISE_UNARY_U8toF32(log)\n\
ELTWISE_UNARY_U8toF32(neg)\n\
ELTWISE_UNARY_U8toF32(mish)\n\
ELTWISE_UNARY_U8toF32(hard_sigmoid)\n\
ELTWISE_UNARY_U8toF32(round)\n\
ELTWISE_UNARY_U8toF32(gelu)\n\
ELTWISE_UNARY_U8toF32(hard_gelu)\n\
ELTWISE_UNARY_U8toF32(selu)\n\
ELTWISE_UNARY_U8toF32(celu)\n\
ELTWISE_UNARY_U8toF32(rcp)\n\
ELTWISE_UNARY_U8toF32(sign)\n\
ELTWISE_UNARY_U8toF32(softsign)\n\
ELTWISE_UNARY_U8toF32(atan)\n\
ELTWISE_UNARY_U8toF32(atanh)\n\
ELTWISE_UNARY_U8toF32(acosh)\n\
ELTWISE_UNARY_U8toF32(inverse_sigmoid)\n\
ELTWISE_UNARY_U8toF32(tan)\n\
\n\
__kernel void neg_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP,\n\
                 float           alpha,\n\
                 float           beta\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src = read_imagei(input, coord);\n\
\n\
    int4 dst = -src;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of eltwise_unary_1_cl*/

static const char erf_cl[] = "float evaluate_polynomial_alpha(float x2)\n\
{\n\
    float4 alpha0 = (float4){-2.72614225801306e-10f, 2.77068142495902e-08f,\n\
                            -2.10102402082508e-06f, -5.69250639462346e-05f};\n\
    float4 alpha1 = (float4){-7.34990630326855e-04f, -2.95459980854025e-03f,\n\
                            -1.60960333262415e-02f, 0};\n\
\n\
    float poly = alpha0.x * x2 + alpha0.y;\n\
    poly = poly * x2 + alpha0.z;\n\
    poly = poly * x2 + alpha0.w;\n\
    poly = poly * x2 + alpha1.x;\n\
    poly = poly * x2 + alpha1.y;\n\
    poly = poly * x2 + alpha1.z;\n\
\n\
    return poly;\n\
}\n\
\n\
float evaluate_polynomial_beta(float x2)\n\
{\n\
    float4 beta0 = (float4){-1.45660718464996e-05f, -2.13374055278905e-04f,\n\
                            -1.68282697438203e-03f, -7.37332916720468e-03f};\n\
    float4 beta1 = (float4){-1.42647390514189e-02f, 0, 0, 0};\n\
\n\
    float poly = beta0.x * x2 + beta0.y;\n\
    poly = poly * x2 + beta0.z;\n\
    poly = poly * x2 + beta0.w;\n\
    poly = poly * x2 + beta1.x;\n\
\n\
    return 1.0f / poly;\n\
}\n\
\n\
float eltwise_unary_erf(float _x)\n\
{\n\
    float x = clamp(_x, -4, 4);\n\
    float x2 = x * x;\n\
\n\
    return x * evaluate_polynomial_alpha(x2) * evaluate_polynomial_beta(x2);\n\
}\n\
\n\
#define ELTWISE_UNARY_F32(func_name) \\\n\
__kernel void func_name##_F32toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x); \\\n\
 \\\n\
    write_imagef(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_F32(erf)\n\
\n\
#define ELTWISE_UNARY_F32_2D(func_name) \\\n\
__kernel void func_name##_F32toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x); \\\n\
 \\\n\
    write_imagef(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_F32_2D(erf)\n\
\n\
#define ELTWISE_UNARY_U8(func_name) \\\n\
__kernel void func_name##_U8toU8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8(erf)\n\
\n\
#define ELTWISE_UNARY_U8_2D(func_name) \\\n\
__kernel void func_name##_U8toU8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8_2D(erf)\n\
"; /* end of erf_cl*/

static const char floordiv_cl[] = "__kernel void floordiv_F32F32toF32\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
    float4 dst  = floor(src0 / src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_F32F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src0 = read_imagef(input, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
    float4 dst  = floor(src0 / src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out = floor(in0 / in1) * outputScale + outputTail;\n\
    int4 dst  = convert_int4(out);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src0 = read_imagei(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out = floor(in0 / in1) * outputScale + outputTail;\n\
    int4 dst  = convert_int4(out);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toU8\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src0 = read_imagei(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    uint4 src0, src1;\n\
    float4 in0, in1, out;\n\
    READ_IMAGEUI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst  = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    uint4 src0 = read_imageui(input, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
    float4 in0, in1, out;\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst  = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8I32toU8\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    uint4 src0;\n\
    int4 src1;\n\
    float4 in0, in1, out;\n\
    READ_IMAGEUI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1);\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8I32toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    uint4 src0 = read_imageui(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0, in1, out;\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1);\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
"; /* end of floordiv_cl*/

static const char gather_cl[] = "__kernel void gather_U8toU8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    uint4 data = read_imageui(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void gather_F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imagef(output, coord, data);\n\
}\n\
\n\
__kernel void gather_I32toI32(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    int4 data = read_imagei(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imagei(output, coord, data);\n\
}\n\
\n\
__kernel void gather_F32toF32(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imagef(output, coord, data);\n\
}\n\
"; /* end of gather_cl*/

static const char gather_array_cl[] = "__kernel void gather_array_U8toU8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    __global uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    uchar data = input_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    __global uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    output_ptr[0] = data;\n\
}\n\
\n\
__kernel void gather_array_F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    __global short* input_ptr = (__global short*)get_image_ptr_from_coord(img1, coord_in.zw);\n\
    short data = input_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    __global short* output_ptr = (__global short*)get_image_ptr_from_coord(img2, coord);\n\
    output_ptr[0] = data;\n\
}\n\
\n\
__kernel void gather_array_I32toI32(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    __global int* input_ptr = (__global int*)get_image_ptr_from_coord(img1, coord_in.zw);\n\
    int data = input_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    __global int* output_ptr = (__global int*)get_image_ptr_from_coord(img2, coord);\n\
    output_ptr[0] = data;\n\
}\n\
\n\
__kernel void gather_array_F32toF32(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 4);\n\
    Image img2 = create_image_from_image2d(output, 4);\n\
    __global float* input_ptr = (__global float*)get_image_ptr_from_coord(img1, coord_in.zw);\n\
    float data = input_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    __global float* output_ptr = (__global float*)get_image_ptr_from_coord(img2, coord);\n\
    output_ptr[0] = data;\n\
}\n\
"; /* end of gather_array_cl*/

static const char gather_batch_cl[] = "__kernel void gather_batch_U8toU8(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        uint4 data = read_imageui(input0, coord_in);\n\
        coord_in.z++;\n\
        write_imageui(output, coord, data);\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
__kernel void gather_batch_F16toF16(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        float4 data = read_imagef(input0, coord_in);\n\
        coord_in.z++;\n\
        write_imagef(output, coord, data);\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
__kernel void gather_batch_I32toI32(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        int4 data = read_imagei(input0, coord_in);\n\
        coord_in.z++;\n\
        write_imagei(output, coord, data);\n\
        coord.z++;\n\
    }\n\
}\n\
\n\
__kernel void gather_batch_F32toF32(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num,\n\
    int batch\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int2 coord_idx = (int2)(gidy, 0);\n\
    int4 coord_in = (int4)(gidx, 0, 0, 0);\n\
    int4 coord = (int4)(gidx, gidz * indices_num + gidy, 0, 0);\n\
    for(; coord_idx.y < batch;)\n\
    {\n\
        int4 indice = read_imagei(input1, coord_idx);\n\
        coord_idx.y++;\n\
        indice.x = indice.x >= 0 ? indice.x : indice.x + axis_num;\n\
        coord_in.y = gidz * axis_num + indice.x;\n\
\n\
        float4 data = read_imagef(input0, coord_in);\n\
        coord_in.z++;\n\
        write_imagef(output, coord, data);\n\
        coord.z++;\n\
    }\n\
}\n\
"; /* end of gather_batch_cl*/

static const char gather_elements_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
_viv_uniform uint width0;\n\
_viv_uniform uint height0;\n\
_viv_uniform uint width1;\n\
_viv_uniform uint height1;\n\
_viv_uniform uint width_out;\n\
_viv_uniform uint height_out;\n\
\n\
#define GATHER_ELEMENTS_AXIS0_2D(name, data_type, read_func, write_func, conv_func) \\\n\
__kernel void gather_elements_axis0_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 int       axis_size \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type data = read_func(input0, (int2)(index, coord.y)); \\\n\
    float4 dst = convert_float4(data) * input_scale + input_tail; \\\n\
    data = conv_func(dst); \\\n\
 \\\n\
    write_func(output, coord, data); \\\n\
}\n\
GATHER_ELEMENTS_AXIS0_2D(F32, float4, read_imagef,  write_imagef,  convert_float4)\n\
GATHER_ELEMENTS_AXIS0_2D(I32, int4,   read_imagei,  write_imagei,  convert_int4_rte)\n\
GATHER_ELEMENTS_AXIS0_2D(U32, uint4,  read_imageui, write_imageui, convert_uint4_rte)\n\
\n\
#define GATHER_ELEMENTS_AXIS0(name, data_type, read_func, write_func, conv_func) \\\n\
__kernel void gather_elements_axis0_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type data = read_func(input0, (int4)(index, coord.yzz)); \\\n\
    float4 dst = convert_float4(data) * input_scale + input_tail; \\\n\
    data = conv_func(dst); \\\n\
 \\\n\
    write_func(output, coord, data); \\\n\
}\n\
GATHER_ELEMENTS_AXIS0(F32, float4, read_imagef,  write_imagef,  convert_float4)\n\
GATHER_ELEMENTS_AXIS0(I32, int4,   read_imagei,  write_imagei,  convert_int4_rte)\n\
GATHER_ELEMENTS_AXIS0(U32, uint4,  read_imageui, write_imageui, convert_uint4_rte)\n\
\n\
#define GATHER_ELEMENTS_AXIS1_2D(name, data_type, read_func, write_func, conv_func) \\\n\
__kernel void gather_elements_axis1_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 int       axis_size \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type data = read_func(input0, (int2)(coord.x, index)); \\\n\
    float4 dst = convert_float4(data) * input_scale + input_tail; \\\n\
    data = conv_func(dst); \\\n\
 \\\n\
    write_func(output, coord, data); \\\n\
}\n\
GATHER_ELEMENTS_AXIS1_2D(F32, float4, read_imagef,  write_imagef,  convert_float4)\n\
GATHER_ELEMENTS_AXIS1_2D(I32, int4,   read_imagei,  write_imagei,  convert_int4_rte)\n\
GATHER_ELEMENTS_AXIS1_2D(U32, uint4,  read_imageui, write_imageui, convert_uint4_rte)\n\
\n\
#define GATHER_ELEMENTS_AXIS1(name, data_type, read_func, write_func, conv_func) \\\n\
__kernel void gather_elements_axis1_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type data = read_func(input0, (int4)(coord.x, index, coord.zz)); \\\n\
    float4 dst = convert_float4(data) * input_scale + input_tail; \\\n\
    data = conv_func(dst); \\\n\
 \\\n\
    write_func(output, coord, data); \\\n\
}\n\
GATHER_ELEMENTS_AXIS1(F32, float4, read_imagef,  write_imagef,  convert_float4)\n\
GATHER_ELEMENTS_AXIS1(I32, int4,   read_imagei,  write_imagei,  convert_int4_rte)\n\
GATHER_ELEMENTS_AXIS1(U32, uint4,  read_imageui, write_imageui, convert_uint4_rte)\n\
\n\
#define GATHER_ELEMENTS_AXIS2(name, data_type, read_func, write_func, conv_func) \\\n\
__kernel void gather_elements_axis2_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
 \\\n\
    int index = read_imagei(input1, coord).x; \\\n\
    int index1 = index + axis_size; \\\n\
    index = index < 0 ? index1 : index; \\\n\
 \\\n\
    data_type data = read_func(input0, (int4)(coord.xy, index, coord.z)); \\\n\
    float4 dst = convert_float4(data) * input_scale + input_tail; \\\n\
    data = conv_func(dst); \\\n\
 \\\n\
    write_func(output, coord, data); \\\n\
}\n\
GATHER_ELEMENTS_AXIS2(F32, float4, read_imagef,  write_imagef,  convert_float4)\n\
GATHER_ELEMENTS_AXIS2(I32, int4,   read_imagei,  write_imagei,  convert_int4_rte)\n\
GATHER_ELEMENTS_AXIS2(U32, uint4,  read_imageui, write_imageui, convert_uint4_rte)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis0_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    Tensor index_tensor = create_tensor_from_image2d_array(input1, 4); \\\n\
    int* index_ptr = (int*)index_tensor.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1 + coord.z * width1 * height1]; \\\n\
 \\\n\
    Tensor input_tensor = create_tensor_from_image2d_array(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_tensor.ptr; \\\n\
    data_type data = input_ptr[index + coord.y * width0 + coord.z * width0 * height0]; \\\n\
 \\\n\
    Tensor output_tensor = create_tensor_from_image2d_array(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_tensor.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out + coord.z * width_out * height_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(F32, float, float*, 4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(I32, int,   int*,   4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0(U8,  uchar, uchar*, 1)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis1_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    Tensor index_tensor = create_tensor_from_image2d_array(input1, 4); \\\n\
    int* index_ptr = (int*)index_tensor.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1 + coord.z * width1 * height1]; \\\n\
 \\\n\
    Tensor input_tensor = create_tensor_from_image2d_array(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_tensor.ptr; \\\n\
    data_type data = input_ptr[coord.x + index * width0 + coord.z * width0 * height0]; \\\n\
 \\\n\
    Tensor output_tensor = create_tensor_from_image2d_array(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_tensor.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out + coord.z * width_out * height_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(F32, float, float*, 4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(I32, int,   int*,   4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1(U8,  uchar, uchar*, 1)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis2_##name##_I32to##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    Tensor index_tensor = create_tensor_from_image2d_array(input1, 4); \\\n\
    int* index_ptr = (int*)index_tensor.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1 + coord.z * width1 * height1]; \\\n\
 \\\n\
    Tensor input_tensor = create_tensor_from_image2d_array(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_tensor.ptr; \\\n\
    data_type data = input_ptr[coord.x + coord.y * width0 + index * width0 * height0]; \\\n\
 \\\n\
    Tensor output_tensor = create_tensor_from_image2d_array(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_tensor.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out + coord.z * width_out * height_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(F32, float, float*, 4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(I32, int,   int*,   4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS2(U8,  uchar, uchar*, 1)\n\
\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis0_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    Image index_img = create_image_from_image2d(input1, 4); \\\n\
    int* index_ptr = (int*)index_img.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1]; \\\n\
 \\\n\
    Image input_img = create_image_from_image2d(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_img.ptr; \\\n\
    data_type data = input_ptr[index + coord.y * width0]; \\\n\
 \\\n\
    Image output_img = create_image_from_image2d(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_img.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(F32, float, float*, 4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(I32, int,   int*,   4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS0_2D(U8,  uchar, uchar*, 1)\n\
\n\
#define GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(name, data_type, data_type_ptr, stride) \\\n\
__kernel void gather_elements_beyond_maxwidth_axis1_##name##_I32to##name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 int             axis_size \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    Image index_img = create_image_from_image2d(input1, 4); \\\n\
    int* index_ptr = (int*)index_img.ptr; \\\n\
    int index = index_ptr[coord.x + coord.y * width1]; \\\n\
 \\\n\
    Image input_img = create_image_from_image2d(input0, stride); \\\n\
    data_type_ptr input_ptr = (data_type_ptr)input_img.ptr; \\\n\
    data_type data = input_ptr[coord.x + index  * width0]; \\\n\
 \\\n\
    Image output_img = create_image_from_image2d(output, stride); \\\n\
    data_type_ptr output_ptr = (data_type_ptr)output_img.ptr; \\\n\
    output_ptr[coord.x + coord.y * width_out] = data; \\\n\
}\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(F32, float, float*, 4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(I32, int,   int*,   4)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(F16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(I16, short, short*, 2)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(I8,  char,  char*,  1)\n\
GATHER_ELEMENTS_BEYOND_MAXWIDTH_AXIS1_2D(U8,  uchar, uchar*, 1)\n\
"; /* end of gather_elements_cl*/

static const char gather_nd_cl[] = "__kernel void gather_nd_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    uint4 data = read_imageui(input0, coord.zw);\n\
    write_imageui(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord.zw);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_I32toI32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    int4 data = read_imagei(input0, coord.zw);\n\
    write_imagei(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F32toF32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord.zw);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
//2D\n\
__kernel void gather_nd_U8toU8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    uint4 data = read_imageui(input0, indice.xy);\n\
    write_imageui(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    float4 data = read_imagef(input0, indice.xy);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_I32toI32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    int4 data = read_imagei(input0, indice.xy);\n\
    write_imagei(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F32toF32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    float4 data = read_imagef(input0, indice.xy);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
"; /* end of gather_nd_cl*/

static const char gather_nd_3d_cl[] = "__kernel void gather_nd_U8toU8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    uint4 data = read_imageui(input0, indice);\n\
    write_imageui(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    float4 data = read_imagef(input0, indice);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_I32toI32_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    int4 data = read_imagei(input0, indice);\n\
    write_imagei(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F32toF32_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    float4 data = read_imagef(input0, indice);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
"; /* end of gather_nd_3d_cl*/

static const char gather_nd_batch_cl[] = "__kernel void gather_nd_batch_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index_num\n\
    int gidz = get_global_id(2);  // batch_num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    int4 indice = read_imagei(input1, coord.wyzw);\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    uint4 data = read_imageui(input0, coord0);\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void gather_nd_batch_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index_num\n\
    int gidz = get_global_id(2);  // batch_num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    int4 indice = read_imagei(input1, coord.wyzw);\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    float4 data = read_imagef(input0, coord0);\n\
    write_imagef(output, coord, data);\n\
}\n\
\n\
__kernel void gather_nd_batch_I8toI8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index_num\n\
    int gidz = get_global_id(2);  // batch_num\n\
\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    int4 indice = read_imagei(input1, coord.wyzw);\n\
    int2 coord0 = (int2)(indice.x * block_size + gidx, gidz);\n\
\n\
    int4 data = read_imagei(input0, coord0);\n\
    write_imagei(output, coord, data);\n\
}\n\
\n\
//2D\n\
__kernel void gather_nd_batch_U8toU8_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index_num\n\
    int gidz = get_global_id(2);  // batch_num\n\
\n\
    int4 coord = (int4)(1, gidy, gidz, 0);\n\
    int4 indice = read_imagei(input1, coord.wyzw);\n\
    int4 indice1 = read_imagei(input1, coord.xyzw);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
    indice.zw = coord.zw;\n\
\n\
    uint4 data = read_imageui(input0, indice);\n\
    coord.x = gidx;\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void gather_nd_batch_F16toF16_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index_num\n\
    int gidz = get_global_id(2);  // batch_num\n\
\n\
    int4 coord = (int4)(1, gidy, gidz, 0);\n\
    int4 indice = read_imagei(input1, coord.wyzw);\n\
    int4 indice1 = read_imagei(input1, coord.xyzw);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
    indice.zw = coord.zw;\n\
\n\
    float4 data = read_imagef(input0, indice);\n\
    coord.x = gidx;\n\
    write_imagef(output, coord, data);\n\
}\n\
\n\
__kernel void gather_nd_batch_I8toI8_2D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // index_num\n\
    int gidz = get_global_id(2);  // batch_num\n\
\n\
    int4 coord = (int4)(1, gidy, gidz, 0);\n\
    int4 indice = read_imagei(input1, coord.wyzw);\n\
    int4 indice1 = read_imagei(input1, coord.xyzw);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
    indice.y = indice1.x;\n\
    indice.zw = coord.zw;\n\
\n\
    int4 data = read_imagei(input0, indice);\n\
    coord.x = gidx;\n\
    write_imagei(output, coord, data);\n\
}\n\
"; /* end of gather_nd_batch_cl*/

static const char globallppool_cl[] = "\n\
#define GLOBALLPPOOL_PROCESS(src_type, dst_type, readimage_type, conv_mode, writeimage_type) \\\n\
    int gidx = get_global_id(0); \\\n\
    int4 coord_out = (int4)(0, 0, gidx, 0); \\\n\
    int4 coord_in  = coord_out; \\\n\
    int h, w; \\\n\
    float sum_of_pow = 0; \\\n\
    dst_type out_data = (dst_type)(0); \\\n\
    src_type in_data; \\\n\
    float in_f32, out_f32; \\\n\
    for (h = 0; h < height; h++) \\\n\
    { \\\n\
        for (w = 0; w < width; w++) \\\n\
        { \\\n\
            coord_in.xy = (int2)(w, h); \\\n\
            in_data = readimage_type(input, coord_in).x; \\\n\
            in_f32 = convert_float(in_data) * inputScale + inputTail; \\\n\
            sum_of_pow += pow(fabs(in_f32),p); \\\n\
        } \\\n\
    } \\\n\
    out_f32 = pow(sum_of_pow, 1.0f / p) * outputScale + outputTail; \\\n\
    out_data.x = conv_mode(out_f32); \\\n\
    writeimage_type(output, coord_out, out_data); \\\n\
\n\
#define TENSOR_GLOBALLPPOOL(src_name, dst_name, src_type, dst_type, readimage_type, conv_mode, writeimage_type) \\\n\
__kernel void globallppool_##src_name##to##dst_name ( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              p, \\\n\
                 int              width, \\\n\
                 int              height, \\\n\
                 float            inputScale, \\\n\
                 float            inputTail, \\\n\
                 float            outputScale, \\\n\
                 float            outputTail) \\\n\
{ \\\n\
    GLOBALLPPOOL_PROCESS(src_type, dst_type, readimage_type, conv_mode, writeimage_type); \\\n\
}\n\
\n\
TENSOR_GLOBALLPPOOL(F32, F32, float, float4, read_imagef, convert_float, write_imagef)\n\
TENSOR_GLOBALLPPOOL(F32, U32, float, uint4,  read_imagef, convert_uint,  write_imageui)\n\
TENSOR_GLOBALLPPOOL(F32, I32, float, int4,   read_imagef, convert_int,   write_imagei)\n\
\n\
TENSOR_GLOBALLPPOOL(U32, U32, uint, uint4,  read_imageui, convert_uint,  write_imageui)\n\
TENSOR_GLOBALLPPOOL(U32, F32, uint, float4, read_imageui, convert_float, write_imagef)\n\
TENSOR_GLOBALLPPOOL(U32, I32, uint, int4,   read_imageui, convert_int,   write_imagei)\n\
\n\
TENSOR_GLOBALLPPOOL(I32, I32, int, int4,    read_imagei, convert_int,   write_imagei)\n\
TENSOR_GLOBALLPPOOL(I32, F32, int, float4, read_imagei, convert_float, write_imagef)\n\
TENSOR_GLOBALLPPOOL(I32, U32, int, uint4,  read_imagei, convert_uint,  write_imageui)\n\
\n\
__kernel void globallppool_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
                 int              p,\n\
                 int              width,\n\
                 int              height,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputTail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord_out = (int4)(1, 1, gidx , 0);\n\
    int4 coord_in  = coord_out;\n\
    int h, w;\n\
    float sum_of_pow = 0;\n\
    float out_data_f32 = 0;\n\
    uint4 dst = (uint4)(0);\n\
    float4 data_f32 = (float4)(0);\n\
    uint4 data;\n\
\n\
    for (h = 0; h < height; h++)\n\
    {\n\
        for (w = 0; w < width; w++)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, data_f32, data, 16);\n\
            sum_of_pow += pow(abs(data_f32.x),p);\n\
        }\n\
    }\n\
    out_data_f32 = pow(sum_of_pow, 1.0f / p);\n\
    _viv_asm(COPY, dst, out_data_f32, 4);\n\
    dst.x = dst.x >> 16;\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
\n\
"; /* end of globallppool_cl*/

static const char group_normalization_f32_cl[] = "__kernel void group_norm_sumsqr_F32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum += data.x;\n\
            sqr += data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_sumsqr_F32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        data = read_imagef(input, coord);\n\
        sum = data.x;\n\
        sqr = data.x * data.x;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_meanvari(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    float group_ratio,\n\
    int group_stride\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
\n\
    float2 sum_sqr = (float2)(0);\n\
    float4 mean_vari = (float4)(0);\n\
\n\
    __local float2 lcl_data[16];\n\
    __local float2 lcl_sum[4];\n\
\n\
    for(; coord.x < group_stride;)\n\
    {\n\
        mean_vari.x += read_imagef(input, coord).x;\n\
        coord.x++;\n\
        mean_vari.y += read_imagef(input, coord).x;\n\
        coord.x+=63;\n\
    }\n\
    lcl_data[lidx] = mean_vari.xy;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx < 4)\n\
    {\n\
        float2 tmpSum = (float2)(0);\n\
        for(int i = lidx; i < 16; i+=4)\n\
        {\n\
            tmpSum += lcl_data[i];\n\
        }\n\
        lcl_sum[lidx] = tmpSum;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx == 0)\n\
    {\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum_sqr += lcl_sum[i];\n\
        }\n\
        mean_vari.xy = sum_sqr * group_ratio;\n\
        mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
        mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
        coord.x = 0;\n\
        write_imagef(output, coord, mean_vari);\n\
        coord.x++;\n\
        float4 data;\n\
        data.x = mean_vari.y;\n\
        write_imagef(output, coord, data);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_F32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = read_imagef(input, coord);\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 dst;\n\
\n\
    dst.x = data.x * scale_vari + bias_val;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_F32toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = read_imagef(input, coord);\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = beta.s0 - scale_vari * mean_vari.s0;\n\
\n\
    float4 dst;\n\
\n\
    dst.x = data.x * scale_vari + bias_val;\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of group_normalization_f32_cl*/

static const char group_normalization_i32_cl[] = "__kernel void group_norm_sumsqr_I32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = convert_float4(read_imagei(input, coord));\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            sqr += (data.x * data.x * e2InScale);\n\
        }\n\
        sum = tmpSum * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_sumsqr_I32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        data = convert_float4(read_imagei(input, coord));\n\
        sum = data.x * input_scale;\n\
        sqr = sum * sum;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_I32toI32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale;\n\
\n\
    int4 dst;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_int4_rte(norm);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_I32toI32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale;\n\
\n\
    int4 dst;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_int4_rte(norm);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
\n\
__kernel void group_norm_I32toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = beta.s0 - scale_vari * mean_vari.s0;\n\
\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
"; /* end of group_normalization_i32_cl*/

static const char group_normalization_u8_cl[] = "__kernel void group_norm_sumsqr_U8(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = convert_float4(read_imageui(input, coord));\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            tmpSqr += data.x * data.x;\n\
        }\n\
        sqr = (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum = (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_sumsqr_U8_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        data = convert_float4(read_imageui(input, coord));\n\
        sum = (data.x - input_zp) * input_scale;\n\
        sqr = sum * sum;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_U8toU8(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 dst;\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_uint4_rte(norm);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_U8toU8_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 dst;\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_uint4_rte(norm);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
\n\
__kernel void group_norm_U8toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
"; /* end of group_normalization_u8_cl*/

static const char grucell_activation_cl[] = "__kernel void grucell_activation(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
\n\
}\n\
"; /* end of grucell_activation_cl*/

static const char grucell_activation_sma_cl[] = "__kernel void grucell_activation_sma(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
\n\
}\n\
"; /* end of grucell_activation_sma_cl*/

static const char grucell_activation_z_h_cl[] = "#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float sigmoid(float x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float hard_sigmoid(float x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float tanh_func(float x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define GRUCELL_ACTIVATION_U8_F32_U8(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_U8_F32toU8_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp, \\\n\
    float output_scale1, float output_zp1) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 h_tm = convert_float4(read_imageui(hstate_in, coord_in.xy)); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 h = h0 + h1; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    float4 out0 = dst * output_scale + output_zp; \\\n\
    float4 out1 = dst * output_scale1 + output_zp1; \\\n\
    uint4 result = convert_uint4_sat_rte(out0); \\\n\
    uint4 result1 = convert_uint4_sat_rte(out1); \\\n\
    write_imageui(output, coord_in.xy, result); \\\n\
    write_imageui(hstate_out, coord_in.xy, result1); \\\n\
}\n\
GRUCELL_ACTIVATION_U8_F32_U8(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_ACTIVATION_F32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_F32_F32toF32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp, \\\n\
    float output_scale1, float output_zp1) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
    float4 h_tm = read_imagef(hstate_in, coord_in.xy); \\\n\
 \\\n\
    float4 h = h0 + h1; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    write_imagef(output, coord_in.xy, dst); \\\n\
    write_imagef(hstate_out, coord_in.xy, dst); \\\n\
}\n\
\n\
GRUCELL_ACTIVATION_F32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_ACTIVATION_I32_F32_I32(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_I32_F32toI32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp, \\\n\
    float output_scale1, float output_zp1) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 h_tm = convert_float4(read_imagei(hstate_in, coord_in.xy)); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 h = h0 + h1; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    float4 out0 = dst * output_scale + output_zp; \\\n\
    float4 out1 = dst * output_scale1 + output_zp1; \\\n\
    int4 result = convert_int4_sat_rte(out0); \\\n\
    int4 result1 = convert_int4_sat_rte(out1); \\\n\
    write_imagei(output, coord_in.xy, result); \\\n\
    write_imagei(hstate_out, coord_in.xy, result1); \\\n\
}\n\
GRUCELL_ACTIVATION_I32_F32_I32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of grucell_activation_z_h_cl*/

static const char grucell_h_times_activation_r_cl[] = "#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float sigmoid(float x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float hard_sigmoid(float x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
#define GRUCELL_H_TIMES_R_U8_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_U8_F32toF32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    float input_scale, float input_tail) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imageui(hstate_in, coord_in.xy)); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h_tm; \\\n\
    write_imagef(output, coord_in.xy, r_times_h); \\\n\
}\n\
GRUCELL_H_TIMES_R_U8_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_H_TIMES_R_U8_F32_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_H_TIMES_R_F32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_F32_F32toF32_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __write_only image2d_t output, \\\n\
    float input_scale, float input_tail) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h_tm = read_imagef(hstate_in, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    float4 r_times_h = r * h_tm; \\\n\
    write_imagef(output, coord_in.xy, r_times_h); \\\n\
}\n\
\n\
GRUCELL_H_TIMES_R_F32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_H_TIMES_R_F32_F32_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_H_TIMES_R_I32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_I32_F32toI32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    float input_scale, float input_tail) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imagei(hstate_in, coord_in.xy)); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h_tm; \\\n\
    write_imagef(output, coord_in.xy, r_times_h); \\\n\
}\n\
GRUCELL_H_TIMES_R_I32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_H_TIMES_R_I32_F32_F32(HARD_SIGMOID, hard_sigmoid)"; /* end of grucell_h_times_activation_r_cl*/

static const char grucell_reset_after_activation_cl[] = "#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float sigmoid(float x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float hard_sigmoid(float x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float tanh_func(float x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
float relu_func(float x)\n\
{\n\
    x = x > 0 ? x : 0;\n\
    return x;\n\
}\n\
\n\
\n\
#define GRUCELL_ACTIVATION_U8_F32_U8(act_name, act_func) \\\n\
__kernel void grucell_reset_after_activation_U8_F32toU8_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imageui(hstate_in, coord_in.xy)); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h1; \\\n\
    float4 h = h0 + r_times_h; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    dst = dst * output_scale + output_zp; \\\n\
    uint4 result = convert_uint4_sat_rte(dst); \\\n\
    write_imageui(output, coord_in.xy, result); \\\n\
    write_imageui(hstate_out, coord_in.xy, result); \\\n\
}\n\
GRUCELL_ACTIVATION_U8_F32_U8(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
GRUCELL_ACTIVATION_U8_F32_U8(RELU, relu_func)\n\
\n\
#define GRUCELL_ACTIVATION_F32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_reset_after_activation_F32_F32toF32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h_tm = read_imagef(hstate_in, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    float4 r_times_h = r * h1; \\\n\
    float4 h = h0 + r_times_h; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    write_imagef(output, coord_in.xy, dst); \\\n\
    write_imagef(hstate_out, coord_in.xy, dst); \\\n\
}\n\
\n\
GRUCELL_ACTIVATION_F32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
GRUCELL_ACTIVATION_F32_F32_F32(RELU, relu_func)\n\
\n\
#define GRUCELL_ACTIVATION_I32_F32_I32(act_name, act_func) \\\n\
__kernel void grucell_reset_after_activation_I32_F32toI32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imagei(hstate_in, coord_in.xy)); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h1; \\\n\
    float4 h = h0 + r_times_h; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    dst = dst * output_scale + output_zp; \\\n\
    int4 result = convert_int4_sat_rte(dst); \\\n\
    write_imagei(output, coord_in.xy, result); \\\n\
    write_imagei(hstate_out, coord_in.xy, result); \\\n\
}\n\
GRUCELL_ACTIVATION_I32_F32_I32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
GRUCELL_ACTIVATION_I32_F32_I32(RELU, relu_func)"; /* end of grucell_reset_after_activation_cl*/

static const char hswish_cl[] = "#define HSWISH_F32_F32_PROCESS() \\\n\
    float4 src, tmp, dst; \\\n\
    src   = read_imagef(input, coord); \\\n\
    tmp   = src + 3; \\\n\
    tmp   = tmp > 0 ? tmp : 0; \\\n\
    tmp   = tmp < 6 ? tmp : 6; \\\n\
    dst   = src * tmp / 6.0f; \\\n\
    write_imagef(output, coord, dst);\n\
\n\
__kernel void hswish_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_F32_F32_PROCESS()\n\
}\n\
\n\
__kernel void hswish_F32toF32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_F32_F32_PROCESS()\n\
}\n\
\n\
\n\
#define HSWISH_U8_U8_PROCESS() \\\n\
    float4 src, tmp, data; \\\n\
    uint4 src0 = read_imageui(input, coord); \\\n\
    src   = convert_float4(src0) * inputScale - inputTail; \\\n\
    tmp   = src + 3; \\\n\
    tmp   = tmp > 0 ? tmp : 0; \\\n\
    tmp   = tmp < 6 ? tmp : 6; \\\n\
    data   = src * tmp / 6.0f; \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
    write_imageui(output, coord, dst);\n\
\n\
__kernel void hswish_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_U8_U8_PROCESS()\n\
}\n\
\n\
__kernel void hswish_U8toU8_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_U8_U8_PROCESS()\n\
}\n\
\n\
\n\
#define HSWISH_I32_I32_PROCESS() \\\n\
    int4 tmp, dst, src; \\\n\
    src   = read_imagei(input, coord); \\\n\
    tmp   = src + 3; \\\n\
    tmp   = tmp > 0 ? tmp : 0; \\\n\
    tmp   = tmp < 6 ? tmp : 6; \\\n\
    dst   = src * tmp / 6; \\\n\
    write_imagei(output, coord, dst);\n\
\n\
__kernel void hswish_I32toI32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_I32_I32_PROCESS()\n\
}\n\
\n\
__kernel void hswish_I32toI32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_I32_I32_PROCESS()\n\
}\n\
"; /* end of hswish_cl*/

static const char instance_normalization_f32_cl[] = "__kernel void instance_norm_sums_F32(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             width,\n\
                 int             height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum = sum + data.x;\n\
            sqr = sqr + data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_sums_F32_2D(\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       width,\n\
                 int       height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum = sum + data.x;\n\
            sqr = sqr + data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_F32toF32(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       bias,\n\
    __read_only  image2d_t       scale,\n\
    __read_only  image2d_t       meanVari,\n\
    __write_only image2d_array_t output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             output_zp,\n\
                 float           output_scale,\n\
                 int             width,\n\
                 int             height,\n\
                 float           inv_multiplier,\n\
                 int             group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 data, dst;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imagef(input, coord);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_F32toF32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t bias,\n\
    __read_only  image2d_t scale,\n\
    __read_only  image2d_t meanVari,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       output_zp,\n\
                 float     output_scale,\n\
                 int       width,\n\
                 int       height,\n\
                 float     inv_multiplier,\n\
                 int       group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 data, dst;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imagef(input, coord);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
    }\n\
}\n\
"; /* end of instance_normalization_f32_cl*/

static const char instance_normalization_i32_cl[] = "__kernel void instance_norm_sums_I32(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             width,\n\
                 int             height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    int4 data;\n\
    float2 sum_x_x2 = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imagei(input, coord);\n\
            coord.y++;\n\
            float in = convert_float(data.x);\n\
            sum_x_x2.x = sum_x_x2.x + in;\n\
            sum_x_x2.y = sum_x_x2.y + in * in;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum_x_x2.x;\n\
    lcl_sqr[lidx] = sum_x_x2.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0, sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_sums_I32_2D(\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       width,\n\
                 int       height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    int4 data;\n\
    float2 sum_x_x2 = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            data = read_imagei(input, coord);\n\
            coord.y++;\n\
            float in = convert_float(data.x);\n\
            sum_x_x2.x = sum_x_x2.x + in;\n\
            sum_x_x2.y = sum_x_x2.y + in * in;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum_x_x2.x;\n\
    lcl_sqr[lidx] = sum_x_x2.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0, sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toI32(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       bias,\n\
    __read_only  image2d_t       scale,\n\
    __read_only  image2d_t       meanVari,\n\
    __write_only image2d_array_t output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             output_zp,\n\
                 float           output_scale,\n\
                 int             width,\n\
                 int             height,\n\
                 float           inv_multiplier,\n\
                 int             group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    int4 data, dst;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_int4_rte(norm);\n\
        write_imagei(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toI32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t bias,\n\
    __read_only  image2d_t scale,\n\
    __read_only  image2d_t meanVari,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       output_zp,\n\
                 float     output_scale,\n\
                 int       width,\n\
                 int       height,\n\
                 float     inv_multiplier,\n\
                 int       group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    int4 data, dst;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_int4_rte(norm);\n\
        write_imagei(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toF32(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       bias,\n\
    __read_only  image2d_t       scale,\n\
    __read_only  image2d_t       meanVari,\n\
    __write_only image2d_array_t output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             output_zp,\n\
                 float           output_scale,\n\
                 int             width,\n\
                 int             height,\n\
                 float           inv_multiplier,\n\
                 int             group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    int4 data;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toF32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t bias,\n\
    __read_only  image2d_t scale,\n\
    __read_only  image2d_t meanVari,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       output_zp,\n\
                 float     output_scale,\n\
                 int       width,\n\
                 int       height,\n\
                 float     inv_multiplier,\n\
                 int       group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_vari;\n\
    bias_val = beta.s0 - scale_vari * mean_vari.s0;\n\
\n\
    int4 data;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
"; /* end of instance_normalization_i32_cl*/

static const char instance_normalization_u8_cl[] = "__kernel void instance_norm_sums_U8(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             width,\n\
                 int             height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    uint4 data;\n\
    float2 sum_x_x2 = 0;\n\
    int2 _sum_x_x2 = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imageui(input, coord);\n\
            coord.y++;\n\
            _sum_x_x2.x = _sum_x_x2.x + data.x;\n\
            _sum_x_x2.y = _sum_x_x2.y + data.x * data.x;\n\
        }\n\
        sum_x_x2 = convert_float2(_sum_x_x2);\n\
    }\n\
    lcl_sum[lidx] = sum_x_x2.x;\n\
    lcl_sqr[lidx] = sum_x_x2.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0, sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_sums_U8_2D(\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       width,\n\
                 int       height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    uint4 data;\n\
    float2 sum_x_x2 = 0;\n\
    int2 _sum_x_x2 = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            data = read_imageui(input, coord);\n\
            coord.y++;\n\
            _sum_x_x2.x = _sum_x_x2.x + data.x;\n\
            _sum_x_x2.y = _sum_x_x2.y + data.x * data.x;\n\
        }\n\
        sum_x_x2 = convert_float2(_sum_x_x2);\n\
    }\n\
    lcl_sum[lidx] = sum_x_x2.x;\n\
    lcl_sqr[lidx] = sum_x_x2.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0, sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toU8(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       bias,\n\
    __read_only  image2d_t       scale,\n\
    __read_only  image2d_t       meanVari,\n\
    __write_only image2d_array_t output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             output_zp,\n\
                 float           output_scale,\n\
                 int             width,\n\
                 int             height,\n\
                 float           inv_multiplier,\n\
                 int             group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data, dst;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_uint4_rte(norm);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toU8_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t bias,\n\
    __read_only  image2d_t scale,\n\
    __read_only  image2d_t meanVari,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       output_zp,\n\
                 float     output_scale,\n\
                 int       width,\n\
                 int       height,\n\
                 float     inv_multiplier,\n\
                 int       group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data, dst;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_uint4_rte(norm);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toF16(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       bias,\n\
    __read_only  image2d_t       scale,\n\
    __read_only  image2d_t       meanVari,\n\
    __write_only image2d_array_t output,\n\
                 float           eps,\n\
                 int             rsFlg,\n\
                 int             output_zp,\n\
                 float           output_scale,\n\
                 int             width,\n\
                 int             height,\n\
                 float           inv_multiplier,\n\
                 int             group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toF16_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t bias,\n\
    __read_only  image2d_t scale,\n\
    __read_only  image2d_t meanVari,\n\
    __write_only image2d_t output,\n\
                 float     eps,\n\
                 int       rsFlg,\n\
                 int       output_zp,\n\
                 float     output_scale,\n\
                 int       width,\n\
                 int       height,\n\
                 float     inv_multiplier,\n\
                 int       group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= inv_multiplier;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
"; /* end of instance_normalization_u8_cl*/

static const char l1norm_cl[] = "#define eps 1e-12\n\
\n\
#define TENSOR_L1NORM_axis0(src_name, dst_name, src_type, dst_type, \\\n\
                      readimage_type, conv_mode, writeimage_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void l1norm_##src_name##to##dst_name##_axis0( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    float4 src_f, dst_f; \\\n\
    float sum = 0; \\\n\
    float rcp_sum = 0; \\\n\
    int4 coord= (int4)(gidx, gidy, gidz, 0); \\\n\
    __local float lcl_sum[16]; \\\n\
    for (; coord.x < axis_size; coord.x += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        sum += fabs(src_f.x); \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    rcp_sum =  1 / (dot(data0, one) + eps); \\\n\
    for (coord.x = gidx; coord.x < axis_size; coord.x += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        dst_f = src_f * rcp_sum; \\\n\
        dst = conv_mode(dst_f * outputscale + outputtail); \\\n\
        writeimage_type(output, coord, dst); \\\n\
    } \\\n\
}\n\
\n\
TENSOR_L1NORM_axis0(F32,F32,float4,float4,read_imagef, convert_float4,write_imagef)\n\
TENSOR_L1NORM_axis0(U32,U32,uint4, uint4, read_imageui,convert_uint4, write_imageui)\n\
TENSOR_L1NORM_axis0(I32,I32,int4,  int4,  read_imagei, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_axis0(F32,U32,float4,uint4, read_imagef, convert_uint4, write_imageui)\n\
TENSOR_L1NORM_axis0(F32,I32,float4,int4,  read_imagef, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_axis0(U32,F32,uint4, float4,read_imageui,convert_float4,write_imagef)\n\
TENSOR_L1NORM_axis0(I32,F32,int4,  float4,read_imagei, convert_float4,write_imagef)\n\
\n\
#define TENSOR_L1NORM_axis1(src_name, dst_name, src_type, dst_type, \\\n\
                      readimage_type, conv_mode, writeimage_type) \\\n\
__kernel __attribute__((reqd_work_group_size(1, 16, 1))) void l1norm_##src_name##to##dst_name##_axis1( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int lidy = get_local_id(1); \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    float4 src_f, dst_f; \\\n\
    float sum = 0; \\\n\
    float rcp_sum = 0; \\\n\
    int4 coord= (int4)(gidx, gidy, gidz, 0); \\\n\
    __local float lcl_sum[16]; \\\n\
    for (; coord.y < axis_size; coord.y += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        sum += fabs(src_f.x); \\\n\
    } \\\n\
    lcl_sum[lidy] = sum; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    rcp_sum =  1 / (dot(data0, one) + eps); \\\n\
    for (coord.y = gidy; coord.y < axis_size; coord.y += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        dst_f = src_f * rcp_sum; \\\n\
        dst = conv_mode(dst_f * outputscale + outputtail); \\\n\
        writeimage_type(output, coord, dst); \\\n\
    } \\\n\
}\n\
\n\
TENSOR_L1NORM_axis1(F32,F32,float4,float4,read_imagef, convert_float4,write_imagef)\n\
TENSOR_L1NORM_axis1(U32,U32,uint4, uint4, read_imageui,convert_uint4, write_imageui)\n\
TENSOR_L1NORM_axis1(I32,I32,int4,  int4,  read_imagei, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_axis1(F32,U32,float4,uint4, read_imagef, convert_uint4, write_imageui)\n\
TENSOR_L1NORM_axis1(F32,I32,float4,int4,  read_imagef, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_axis1(U32,F32,uint4, float4,read_imageui,convert_float4,write_imagef)\n\
TENSOR_L1NORM_axis1(I32,F32,int4,  float4,read_imagei, convert_float4,write_imagef)\n\
\n\
#define TENSOR_L1NORM_axis2(src_name, dst_name, src_type, dst_type, \\\n\
                      readimage_type, conv_mode, writeimage_type) \\\n\
__kernel __attribute__((reqd_work_group_size(1, 1, 16))) void l1norm_##src_name##to##dst_name##_axis2( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int lidz = get_local_id(2); \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    float4 src_f, dst_f; \\\n\
    float sum = 0; \\\n\
    float rcp_sum = 0; \\\n\
    int4 coord= (int4)(gidx, gidy, gidz, 0); \\\n\
    __local float lcl_sum[16]; \\\n\
    for (; coord.z < axis_size; coord.z += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        sum += fabs(src_f.x); \\\n\
    } \\\n\
    lcl_sum[lidz] = sum; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    rcp_sum =  1 / (dot(data0, one) + eps); \\\n\
    for (coord.z = gidz; coord.z < axis_size; coord.z += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        dst_f = src_f * rcp_sum; \\\n\
        dst = conv_mode(dst_f * outputscale + outputtail); \\\n\
        writeimage_type(output, coord, dst); \\\n\
    } \\\n\
}\n\
\n\
TENSOR_L1NORM_axis2(F32,F32,float4,float4,read_imagef, convert_float4,write_imagef)\n\
TENSOR_L1NORM_axis2(U32,U32,uint4, uint4, read_imageui,convert_uint4, write_imageui)\n\
TENSOR_L1NORM_axis2(I32,I32,int4,  int4,  read_imagei, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_axis2(F32,U32,float4,uint4, read_imagef, convert_uint4, write_imageui)\n\
TENSOR_L1NORM_axis2(F32,I32,float4,int4,  read_imagef, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_axis2(U32,F32,uint4, float4,read_imageui,convert_float4,write_imagef)\n\
TENSOR_L1NORM_axis2(I32,F32,int4,  float4,read_imagei, convert_float4,write_imagef)\n\
\n\
#define TENSOR_L1NORM_2D_axis0(src_name, dst_name, src_type, dst_type,\\\n\
          readimage_type, conv_mode, writeimage_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void l1norm_##src_name##to##dst_name##_2D_axis0( \\\n\
    __read_only  image2d_t        input, \\\n\
    __write_only image2d_t        output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    float4 src_f, dst_f; \\\n\
    float sum = 0; \\\n\
    float rcp_sum = 0; \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    __local float lcl_sum[16]; \\\n\
    for (; coord.x < axis_size; coord.x += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        sum += fabs(src_f.x); \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    rcp_sum = 1 / (dot(data0, one) + eps); \\\n\
    for (coord.x = gidx; coord.x < axis_size; coord.x += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        dst_f = src_f * rcp_sum; \\\n\
        dst = conv_mode(dst_f * outputscale + outputtail); \\\n\
        writeimage_type(output, coord, dst); \\\n\
    } \\\n\
}\n\
\n\
TENSOR_L1NORM_2D_axis0(F32,F32,float4,float4,read_imagef, convert_float4,write_imagef)\n\
TENSOR_L1NORM_2D_axis0(U32,U32,uint4, uint4, read_imageui,convert_uint4, write_imageui)\n\
TENSOR_L1NORM_2D_axis0(I32,I32,int4,  int4,  read_imagei, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_2D_axis0(F32,U32,float4,uint4, read_imagef, convert_uint4, write_imageui)\n\
TENSOR_L1NORM_2D_axis0(F32,I32,float4,int4,  read_imagef, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_2D_axis0(U32,F32,uint4, float4,read_imageui,convert_float4,write_imagef)\n\
TENSOR_L1NORM_2D_axis0(I32,F32,int4,  float4,read_imagei, convert_float4,write_imagef)\n\
\n\
\n\
#define TENSOR_L1NORM_2D_axis1(src_name, dst_name, src_type, dst_type,\\\n\
             readimage_type, conv_mode, writeimage_type) \\\n\
__kernel __attribute__((reqd_work_group_size(1, 16, 1))) void l1norm_##src_name##to##dst_name##_2D_axis1( \\\n\
    __read_only  image2d_t        input, \\\n\
    __write_only image2d_t        output, \\\n\
                 float            inputZp, \\\n\
                 float            outputscale, \\\n\
                 float            outputtail, \\\n\
                 int              axis, \\\n\
                 int              axis_size) \\\n\
{ \\\n\
    int lidy = get_local_id(1); \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    float4 src_f, dst_f; \\\n\
    float sum = 0; \\\n\
    float rcp_sum = 0; \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    __local float lcl_sum[16]; \\\n\
    for (; coord.y < axis_size; coord.y += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        sum += fabs(src_f.x); \\\n\
    } \\\n\
    lcl_sum[lidy] = sum; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    rcp_sum = 1 / (dot(data0, one) + eps); \\\n\
    for (coord.y = gidy; coord.y < axis_size; coord.y += 16) \\\n\
    { \\\n\
        src = readimage_type(input, coord); \\\n\
        src_f = convert_float4(src) - inputZp; \\\n\
        dst_f = src_f * rcp_sum; \\\n\
        dst = conv_mode(dst_f * outputscale + outputtail); \\\n\
        writeimage_type(output, coord, dst); \\\n\
    } \\\n\
}\n\
\n\
TENSOR_L1NORM_2D_axis1(F32,F32,float4,float4,read_imagef, convert_float4,write_imagef)\n\
TENSOR_L1NORM_2D_axis1(U32,U32,uint4, uint4, read_imageui,convert_uint4, write_imageui)\n\
TENSOR_L1NORM_2D_axis1(I32,I32,int4,  int4,  read_imagei, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_2D_axis1(F32,U32,float4,uint4, read_imagef, convert_uint4, write_imageui)\n\
TENSOR_L1NORM_2D_axis1(F32,I32,float4,int4,  read_imagef, convert_int4,  write_imagei)\n\
TENSOR_L1NORM_2D_axis1(U32,F32,uint4, float4,read_imageui,convert_float4,write_imagef)\n\
TENSOR_L1NORM_2D_axis1(I32,F32,int4,  float4,read_imagei, convert_float4,write_imagef)"; /* end of l1norm_cl*/

static const char l2normalizescale_axis0_cl[] = "\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void l2normalizescale_axis0_F32_F32toF32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int2 coord_scale = (int2)(gidx, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        src = read_imagef(input, coord);\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for (coord.x = gidx; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        coord_scale.x = coord.x;\n\
        src         = read_imagef(input, coord);\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void l2normalizescale_axis0_U8_F32toU8_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int2 coord_scale = (int2)(gidx, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for (coord.x = gidx; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        coord_scale.x = coord.x;\n\
        src         = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void l2normalizescale_axis0_I32_F32toI32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int2 coord_scale = (int2)(gidx, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        src = convert_float4(read_imagei(input, coord))  * inputScale + inputTail;\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for (coord.x = gidx; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        coord_scale.x = coord.x;\n\
        src         = convert_float4(read_imagei(input, coord))  * inputScale + inputTail;\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        int4 dst = convert_int4_rte(result * outputScale + outputZP);\n\
        write_imagei(output, coord, dst);\n\
    }\n\
}\n\
"; /* end of l2normalizescale_axis0_cl*/

static const char l2normalizescale_axis1_cl[] = "\n\
__kernel __attribute__((reqd_work_group_size(1, 16, 1))) void l2normalizescale_axis1_F32_F32toF32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(1);\n\
    int gidy = get_global_id(1);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(get_global_id(0), gidy );\n\
    int2 coord_scale = (int2)(gidy, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        src = read_imagef(input, coord);\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for (coord.y = gidy; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        coord_scale.x = coord.y;\n\
        src         = read_imagef(input, coord);\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(1, 16, 1))) void l2normalizescale_axis1_U8_F32toU8_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(1);\n\
    int gidy = get_global_id(1);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(get_global_id(0), gidy );\n\
    int2 coord_scale = (int2)(gidy, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for (coord.y = gidy; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        coord_scale.x = coord.y;\n\
        src         = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(1, 16, 1))) void l2normalizescale_axis1_I32_F32toI32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(1);\n\
    int gidy = get_global_id(1);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(get_global_id(0), gidy );\n\
    int2 coord_scale = (int2)(gidy, 0);\n\
    __local float lcl_sum[16];\n\
    for (; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        src = convert_float4(read_imagei(input, coord)) * inputScale + inputTail;\n\
        pSum = pSum + src.x * src.x;\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for (coord.y = gidy; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        coord_scale.x = coord.y;\n\
        src         = convert_float4(read_imagei(input, coord)) * inputScale + inputTail;\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        int4 dst = convert_int4_rte(result * outputScale + outputZP);\n\
        write_imagei(output, coord, dst);\n\
    }\n\
}"; /* end of l2normalizescale_axis1_cl*/

static const char layer_normalization_cl[] = "\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layer_norm_F32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float e2InScale,\n\
    float scale_inOut,\n\
    float sumZpScale,\n\
    float zp2ScaleE2,\n\
    float sumZpScaleE2,\n\
    int width,\n\
    int height,\n\
    float dim_ratio\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(lidx, get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 data, dst;\n\
    float2 sumSqr = (float2)(0);\n\
    float scale_vari, bias_val;\n\
    __local float2 local_sum[16];\n\
\n\
    for(; coord.x < width;)\n\
    {\n\
        data = read_imagef(input, coord);\n\
        coord.x += 16;\n\
        sumSqr.x += data.x;\n\
        sumSqr.y += data.x * data.x;\n\
    }\n\
    local_sum[lidx] = sumSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx == 0)\n\
    {\n\
        for(int i = 1; i < 16; i++)\n\
        {\n\
            sumSqr += local_sum[i];\n\
        }\n\
        local_sum[0] = sumSqr;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    sumSqr = local_sum[0] * dim_ratio;\n\
    sumSqr.s1 = sumSqr.s1 - sumSqr.s0 * sumSqr.s0 + eps;\n\
    sumSqr.s1 = rsqrt(sumSqr.s1);\n\
\n\
    for(coord.x = lidx; coord.x < width;)\n\
    {\n\
        float4 gamma = read_imagef(scale, coord.xw);\n\
        float4 beta  = read_imagef(bias, coord.xw);\n\
        data = read_imagef(input, coord);\n\
\n\
        scale_vari = gamma.s0 * sumSqr.s1;\n\
        bias_val = (beta.s0 - scale_vari * sumSqr.s0);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
        coord.x += 16;\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layer_norm_U8toU8(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float e2InScale,\n\
    float scale_inOut,\n\
    float sumZpScale,\n\
    float zp2ScaleE2,\n\
    float sumZpScaleE2,\n\
    int width,\n\
    int height,\n\
    float dim_ratio\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(lidx, get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 data, dst;\n\
    float2 sumSqr;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float scale_vari, bias_val;\n\
    __local uint local_sum[1];\n\
    __local uint local_sqr[1];\n\
\n\
    if(lidx == 0)\n\
    {\n\
        local_sum[0] = 0;\n\
        local_sqr[0] = 0;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    for(; coord.x < width;)\n\
    {\n\
        data = read_imageui(input, coord);\n\
        coord.x+=16;\n\
        tmpSum += data.x;\n\
        tmpSqr += data.x * data.x;\n\
    }\n\
    atom_add(local_sum, tmpSum);\n\
    atom_add(local_sqr, tmpSqr);\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    tmpSum = local_sum[0];\n\
    tmpSqr = local_sqr[0];\n\
    //sumSqr.x = ((float)tmpSum - width * input_zp) * input_scale;\n\
    //sumSqr.y = ((float)tmpSqr - 2 * input_zp * (float)tmpSum + width * input_zp * input_zp) * e2InScale;\n\
    sumSqr.x = (float)tmpSum * input_scale - sumZpScale;\n\
    sumSqr.y = (float)tmpSqr * e2InScale - zp2ScaleE2 * (float)tmpSum + sumZpScaleE2;\n\
\n\
    sumSqr *= dim_ratio;\n\
    sumSqr.s1 = sumSqr.s1 - sumSqr.s0 * sumSqr.s0 + eps;\n\
    sumSqr.s1 = rsqrt(sumSqr.s1);\n\
\n\
    for(coord.x = lidx; coord.x < width;)\n\
    {\n\
        float4 gamma = read_imagef(scale, coord.xw);\n\
        float4 beta  = read_imagef(bias, coord.xw);\n\
        data = read_imageui(input, coord);\n\
\n\
        scale_vari = gamma.s0 * sumSqr.s1;\n\
        float alpha = scale_inOut * scale_vari;\n\
        bias_val = (beta.s0 - scale_vari * sumSqr.s0) * output_scale + output_zp;\n\
\n\
        float tmpVal = data.x - input_zp;\n\
\n\
        float4 norm;\n\
        norm.x = tmpVal * alpha + bias_val;\n\
        dst = convert_uint4_rte(norm);\n\
        write_imageui(output, coord, dst);\n\
        coord.x+=16;\n\
    }\n\
}\n\
"; /* end of layer_normalization_cl*/

static const char log_softmax_axis0_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define rlogE    (0.693147182f)\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_axis0_F32toF32(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int width = get_image_width(input);\n\
    int4 coord_in = (int4)(0, y, z, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_F32toF32_2D(\n\
    __read_only   image2d_t input,\n\
    __write_only  image2d_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int width = get_image_width(input);\n\
    int2 coord_in = (int2)(0, y);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.0f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_U8toU8(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int width = get_image_width(input);\n\
    int4 coord_in = (int4)(0, y, z, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_U8toU8_2D(\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int width = get_image_width(input);\n\
    int2 coord_in = (int2)(0, y);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x)*scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_BF16toBF16(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int width = get_image_width(input);\n\
    int4 coord_in = (int4)(0, y, z, 0);\n\
    float4 maxValue, src, dst = {0.0};\n\
    uint4 data, val, out;\n\
\n\
    data = read_imageui(input, coord_in);\n\
    data = data << 16;\n\
    _viv_asm(COPY, maxValue, data, 16);\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.x++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.x++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        _viv_asm(COPY, val, dst, 16);\n\
        out = val >> 16;\n\
        write_imageui(output, coord_in, out);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_BF16toBF16_2D(\n\
    __read_only   image2d_t input,\n\
    __write_only  image2d_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int width = get_image_width(input);\n\
    int2 coord_in = (int2)(0, y);\n\
    float4 maxValue, src, dst = {0.0};\n\
    uint4 data, val, out;\n\
\n\
    data = read_imageui(input, coord_in);\n\
    data = data << 16;\n\
    _viv_asm(COPY, maxValue, data, 16);\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.x++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    float sum = 0.0f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.x++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        _viv_asm(COPY, val, dst, 16);\n\
        out = val >> 16;\n\
        write_imageui(output, coord_in, out);\n\
        coord_in.x++;\n\
    }\n\
}\n\
#undef rlogE\n\
"; /* end of log_softmax_axis0_cl*/

static const char log_softmax_axis1_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define rlogE    (0.693147182f)\n\
\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_axis1_F32toF32(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
    int axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int height = get_image_height(input);\n\
    int4 coord_in = (int4)(x, 0, z, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_F32toF32_2D(\n\
    __read_only   image2d_t input,\n\
    __write_only  image2d_t output,\n\
    int axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int height = get_image_height(input);\n\
    int2 coord_in = (int2)(x, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.0f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = 1.0f * LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_U8toU8(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int height = get_image_height(input);\n\
    int4 coord_in = (int4)(x, 0, z, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_U8toU8_2D(\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
    int axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int height = get_image_height(input);\n\
    int2 coord_in = (int2)(x, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x)*scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16oBF16(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
        int axis, float beta,\n\
        float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int height = get_image_height(input);\n\
    int4 coord_in = (int4)(x, 0, z, 0);\n\
    float4 maxValue, src, dst = {0.0};\n\
    uint4 data, val, out;\n\
\n\
    data = read_imageui(input, coord_in);\n\
    data = data << 16;\n\
    _viv_asm(COPY, maxValue, data, 16);\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.y++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.y++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
\n\
        _viv_asm(COPY, val, dst, 16);\n\
        out = val >> 16;\n\
\n\
        write_imageui(output, coord_in, out);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toBF16_2D(\n\
    __read_only   image2d_t input,\n\
    __write_only  image2d_t output,\n\
        int axis, float beta,\n\
        float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int height = get_image_height(input);\n\
    int2 coord_in = (int2)(x, 0);\n\
    float4 maxValue, src, dst = {0.0};\n\
    uint4 data, val, out;\n\
\n\
    data = read_imageui(input, coord_in);\n\
    data = data << 16;\n\
    _viv_asm(COPY, maxValue, data, 16);\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.y++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    float sum = 0.0f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.y++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    float logSum = 1.0f * LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        _viv_asm(COPY, val, dst, 16);\n\
        out = val >> 16;\n\
        write_imageui(output, coord_in, out);\n\
        coord_in.y++;\n\
    }\n\
}\n\
#undef rlogE\n\
"; /* end of log_softmax_axis1_cl*/

static const char log_softmax_axis2_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#define rlogE    (0.693147182f)\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_axis2_F32toF32\n\
    (\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
                            int   axis,\n\
                            float beta,\n\
                            float scale,\n\
                            float scaleOut,\n\
                            float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int depth = get_image_array_size(input);\n\
    int4 coord_in = (int4)(x, y, 0, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.z = 1; coord_in.z < depth; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.z++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.z++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.z++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
                            int   axis,\n\
                            float beta,\n\
                            float scale,\n\
                            float scaleOut,\n\
                            float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int depth = get_image_array_size(input);\n\
    int4 coord_in = (int4)(x, y, 0, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.z = 1; coord_in.z < depth; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.z++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.z++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.z++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis2_BF16toBF16\n\
    (\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
                            int   axis,\n\
                            float beta,\n\
                            float scale,\n\
                            float scaleOut,\n\
                            float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int depth = get_image_array_size(input);\n\
    int4 coord_in = (int4)(x, y, 0, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
    uint4 data, val, out;\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    data = read_imageui(input, coord_in);\n\
    data = data << 16;\n\
    _viv_asm(COPY, maxValue, data, 16);\n\
    for (coord_in.z = 1; coord_in.z < depth; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.z++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        coord_in.z++;\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        data = read_imageui(input, coord_in);\n\
        data = data << 16;\n\
        _viv_asm(COPY, src, data, 16);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        _viv_asm(COPY, val, dst, 16);\n\
        out = val >> 16;\n\
        write_imageui(output, coord_in, out);\n\
        coord_in.z++;\n\
    }\n\
}\n\
#undef rlogE\n\
"; /* end of log_softmax_axis2_cl*/

static const char log_softmax_exceed_axis0_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
\n\
\n\
#define rlogE    (0.693147182f)\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_exceed_axis0_F32toF32(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int z = get_global_id(1);\n\
    int4 coord_in = (int4)(0, 0, z, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            src = read_imagef(input, coord_in);\n\
            maxValue = maxValue > src ? maxValue : src;\n\
        }\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            src = read_imagef(input, coord_in);\n\
            sum += exp2((src.x - maxValue.x) * scale);\n\
        }\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            src = read_imagef(input, coord_in);\n\
            dst.x = (src.x - maxValue.x) * beta - logSum;\n\
            write_imagef(output, coord_in, dst);\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_exceed_axis0_U8toU8(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int z = get_global_id(1);\n\
    int4 coord_in = (int4)(0, 0, z, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            src = convert_float4(read_imageui(input, coord_in));\n\
            maxValue = maxValue > src ? maxValue : src;\n\
        }\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            src = convert_float4(read_imageui(input, coord_in));\n\
            sum += exp2((src.x - maxValue.x) * scale);\n\
        }\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            src = convert_float4(read_imageui(input, coord_in));\n\
\n\
            dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
            write_imageui(output, coord_in, dst);\n\
        }\n\
    }\n\
}\n\
\n\
\n\
__kernel void log_softmax_exceed_axis0_BF16toBF16(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
    int   axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int z = get_global_id(1);\n\
    int4 coord_in = (int4)(0, 0, z, 0);\n\
    float4 maxValue, src, dst = {0.0};\n\
    uint4 data, val, out;\n\
\n\
    data = read_imageui(input, coord_in);\n\
    data = data << 16;\n\
    _viv_asm(COPY, maxValue, data, 16);\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, src, data, 16);\n\
\n\
            maxValue = maxValue > src ? maxValue : src;\n\
        }\n\
    }\n\
\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, src, data, 16);\n\
\n\
            sum += exp2((src.x - maxValue.x) * scale);\n\
        }\n\
    }\n\
\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; coord_in.x++)\n\
    {\n\
        for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
        {\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, src, data, 16);\n\
\n\
            dst.x = (src.x - maxValue.x) * beta - logSum;\n\
            _viv_asm(COPY, val, dst, 16);\n\
            out = val >> 16;\n\
            write_imageui(output, coord_in, out);\n\
        }\n\
    }\n\
}\n\
#undef rlogE\n\
"; /* end of log_softmax_exceed_axis0_cl*/

static const char log_softmax_exceed_axis1_cl[] = "#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int height;\n\
_viv_uniform int depth;\n\
\n\
#define rlogE    (0.693147182f)\n\
\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_exceed_axis1_F32toF32(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
    int axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int4 coord_in = (int4)(x, 0, 0, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            src = read_imagef(input, coord_in);\n\
            maxValue = maxValue > src ? maxValue : src;\n\
        }\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            src = read_imagef(input, coord_in);\n\
            sum += exp2((src.x - maxValue.x) * scale);\n\
        }\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            src = read_imagef(input, coord_in);\n\
\n\
            dst.x = (src.x - maxValue.x) * beta - logSum;\n\
            write_imagef(output, coord_in, dst);\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_exceed_axis1_U8toU8(\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int axis, float beta,\n\
    float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int4 coord_in = (int4)(x, 0, 0, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            src = convert_float4(read_imageui(input, coord_in));\n\
\n\
            maxValue = maxValue > src ? maxValue : src;\n\
        }\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            src = convert_float4(read_imageui(input, coord_in));\n\
\n\
            sum += exp2((src.x - maxValue.x) * scale);\n\
        }\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            src = convert_float4(read_imageui(input, coord_in));\n\
\n\
            dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
            write_imageui(output, coord_in, dst);\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_exceed_axis1_BF16oBF16(\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
        int axis, float beta,\n\
        float scale, float scaleOut, float zpOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int4 coord_in = (int4)(x, 0, 0, 0);\n\
    float4 maxValue, src, dst = {0.0};\n\
    uint4 data, val, out;\n\
\n\
    data = read_imageui(input, coord_in);\n\
    data = data << 16;\n\
    _viv_asm(COPY, maxValue, data, 16);\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, src, data, 16);\n\
\n\
            maxValue = maxValue > src ? maxValue : src;\n\
        }\n\
    }\n\
\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, src, data, 16);\n\
\n\
            sum += exp2((src.x - maxValue.x) * scale);\n\
        }\n\
    }\n\
\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; coord_in.y++)\n\
    {\n\
        for (coord_in.z = 0; coord_in.z < depth; coord_in.z++)\n\
        {\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, src, data, 16);\n\
\n\
            dst.x = (src.x - maxValue.x) * beta - logSum;\n\
\n\
            _viv_asm(COPY, val, dst, 16);\n\
            out = val >> 16;\n\
\n\
            write_imageui(output, coord_in, out);\n\
        }\n\
    }\n\
}\n\
\n\
#undef rlogE\n\
"; /* end of log_softmax_exceed_axis1_cl*/

static const char logical_not_cl[] = "__kernel void logical_not_I8toI8(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src   = read_imagei(input, coord);\n\
    int4 dst   = !src;\n\
    dst.x = dst.x & 1;\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void logical_not_I8toI8_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src   = read_imagei(input, coord);\n\
    int4 dst   = !src;\n\
    dst.x = dst.x & 1;\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of logical_not_cl*/

static const char logical_ops_cl[] = "#define TENSORLOGICAL(name, lgc_op, lgc_op2) \\\n\
__kernel void logical_##name##_I8toI8( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_array_t   input1, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 src0; \\\n\
    int4 src1; \\\n\
    READ_IMAGEI_2DARRAY(src0, input, coord); \\\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord); \\\n\
    int4 dst  = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    dst.x = dst.x & 1; \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
TENSORLOGICAL(or,  ||, )\n\
TENSORLOGICAL(and, &&, )\n\
TENSORLOGICAL(xor, ^,  !!)\n\
\n\
\n\
#define TENSORLOGICAL_2D(name, lgc_op, lgc_op2) \\\n\
__kernel void logical_##name##_I8toI8_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int4 src0 = read_imagei(input, coord); \\\n\
    int4 src1 = read_imagei(input1, coord); \\\n\
    int4 dst  = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    dst.x = dst.x & 1; \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
TENSORLOGICAL_2D(or,  ||, )\n\
TENSORLOGICAL_2D(and, &&, )\n\
TENSORLOGICAL_2D(xor, ^,  !!)\n\
"; /* end of logical_ops_cl*/

static const char lppool_cl[] = "\n\
#define LPPOOL_PROCESS(src_type, dst_type, readimage_type, conv_mode, writeimage_type) \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int hstart = gidy * stride_y - pad_top; \\\n\
    int wstart = gidx * stride_x - pad_left; \\\n\
    int hend = min(hstart + ksize_y, height); \\\n\
    int wend = min(wstart + ksize_x, width); \\\n\
    int4 coord_out = (int4)(gidx, gidy, get_global_id(2), 0); \\\n\
    int4 coord_in  = coord_out; \\\n\
    int h, w; \\\n\
    float sum_of_pow = 0; \\\n\
    dst_type out_data = (dst_type)(0); \\\n\
    src_type in_data; \\\n\
    float in_f32, out_f32; \\\n\
    hstart = max(hstart, 0); \\\n\
    wstart = max(wstart, 0); \\\n\
    for (h = hstart; h < hend; h++) \\\n\
    { \\\n\
        for (w = wstart; w < wend; w++) \\\n\
        { \\\n\
            coord_in.xy = (int2)(w, h); \\\n\
            in_data = readimage_type(input, coord_in).x; \\\n\
            in_f32 = convert_float(in_data) * inputScale + inputTail; \\\n\
            sum_of_pow += pow(fabs(in_f32),p); \\\n\
        } \\\n\
    } \\\n\
    out_f32 = pow(sum_of_pow, 1.0f / p) * outputScale + outputTail; \\\n\
    out_data.x = conv_mode(out_f32); \\\n\
    writeimage_type(output, coord_out, out_data); \\\n\
\n\
#define TENSOR_LPPOOL(src_name, dst_name, src_type, dst_type, readimage_type, conv_mode, writeimage_type) \\\n\
__kernel void lppool_##src_name##to##dst_name ( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              ksize_x, \\\n\
                 int              ksize_y, \\\n\
                 int              stride_x, \\\n\
                 int              stride_y, \\\n\
                 int              pad_left, \\\n\
                 int              pad_top, \\\n\
                 int              p, \\\n\
                 int              width, \\\n\
                 int              height, \\\n\
                 float            inputScale, \\\n\
                 float            inputTail, \\\n\
                 float            outputScale, \\\n\
                 float            outputTail) \\\n\
{ \\\n\
    LPPOOL_PROCESS(src_type, dst_type, readimage_type, conv_mode, writeimage_type); \\\n\
}\n\
\n\
TENSOR_LPPOOL(F32, F32, float, float4, read_imagef, convert_float, write_imagef)\n\
TENSOR_LPPOOL(F32, U32, float, uint4,  read_imagef, convert_uint,  write_imageui)\n\
TENSOR_LPPOOL(F32, I32, float, int4,   read_imagef, convert_int,   write_imagei)\n\
\n\
TENSOR_LPPOOL(U32, U32, uint, uint4,  read_imageui, convert_uint,  write_imageui)\n\
TENSOR_LPPOOL(U32, F32, uint, float4, read_imageui, convert_float, write_imagef)\n\
TENSOR_LPPOOL(U32, I32, uint, int4,   read_imageui, convert_int,   write_imagei)\n\
\n\
TENSOR_LPPOOL(I32, I32, int, int4,    read_imagei, convert_int,   write_imagei)\n\
TENSOR_LPPOOL(I32, F32, int, float4, read_imagei, convert_float, write_imagef)\n\
TENSOR_LPPOOL(I32, U32, int, uint4,  read_imagei, convert_uint,  write_imageui)\n\
\n\
__kernel void lppool_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
                 int              ksize_x,\n\
                 int              ksize_y,\n\
                 int              stride_x,\n\
                 int              stride_y,\n\
                 int              pad_left,\n\
                 int              pad_top,\n\
                 int              p,\n\
                 int              width,\n\
                 int              height,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputTail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int4 coord_out = (int4)(gidx, gidy, get_global_id(2), 0);\n\
    int4 coord_in  = coord_out;\n\
    int h, w;\n\
    float sum_of_pow = 0;\n\
    float out_data_f32 = 0;\n\
    uint4 dst = (uint4)(0);\n\
    float4 data_f32 = (float4)(0);\n\
    uint4 data;\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
\n\
    for (h = hstart; h < hend; h++)\n\
    {\n\
        for (w = wstart; w < wend; w++)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            data = read_imageui(input, coord_in);\n\
            data = data << 16;\n\
            _viv_asm(COPY, data_f32, data, 16);\n\
            sum_of_pow += pow(abs(data_f32.x),p);\n\
        }\n\
    }\n\
    out_data_f32 = pow(sum_of_pow, 1.0f / p);\n\
    _viv_asm(COPY, dst, out_data_f32, 4);\n\
    dst.x = dst.x >> 16;\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
\n\
"; /* end of lppool_cl*/

static const char lstmunit_activation_BP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_BP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_BP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_F32_cl*/

static const char lstmunit_activation_BP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_BP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_BP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_U8_cl*/

static const char lstmunit_activation_B_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_B_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_B_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_F32TOU8(HARD_SIGMOID, hard_sigmoid)"; /* end of lstmunit_activation_B_F32_cl*/

static const char lstmunit_activation_B_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_B_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_B_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_B_U8_cl*/

static const char lstmunit_activation_CBP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CBP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CBP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_F32_cl*/

static const char lstmunit_activation_CBP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CBP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CBP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_U8_cl*/

static const char lstmunit_activation_CB_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_F32_cl*/

static const char lstmunit_activation_CB_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_U8_cl*/

static const char lstmunit_activation_CLP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CLP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CLP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CLP_F32_cl*/

static const char lstmunit_activation_CLP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CLP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CLP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CLP_U8_cl*/

static const char lstmunit_activation_CL_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CL_F32_cl*/

static const char lstmunit_activation_CL_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CL_U8_cl*/

static const char lstmunit_activation_CSP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CSP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CSP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_F32_cl*/

static const char lstmunit_activation_CSP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CSP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CSP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_U8_cl*/

static const char lstmunit_activation_CS_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CS_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CS_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
"; /* end of lstmunit_activation_CS_F32_cl*/

static const char lstmunit_activation_CS_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CS_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CS_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
"; /* end of lstmunit_activation_CS_U8_cl*/

static const char lstmunit_activation_LP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_LP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_LP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t        input_i_conv, \\\n\
    __read_only  image2d_t        input_f_conv, \\\n\
    __read_only  image2d_t        input_c_conv, \\\n\
    __read_only  image2d_t        input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 w0, w1, w2, w3; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 * w0 + b0; \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_LP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_LP_F32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_LP_F32_cl*/

static const char lstmunit_activation_L_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_L_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_L_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t        input_i_conv, \\\n\
    __read_only  image2d_t        input_f_conv, \\\n\
    __read_only  image2d_t        input_c_conv, \\\n\
    __read_only  image2d_t        input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 w0, w1, w2, w3; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 * w0 + b0; \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_L_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_L_F32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_L_F32_cl*/

static const char lstmunit_activation_SP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_F32_cl*/

static const char lstmunit_activation_SP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_U8_cl*/

static const char lstmunit_activation_S_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_S_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_S_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_F32_cl*/

static const char lstmunit_activation_S_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_S_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_S_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_U8_cl*/

static const char matrixmul_cl[] = "__kernel void gemm_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.zy);\n\
        tempB0 = read_imagef(inputB, coord.xz);\n\
        coord.z++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_F32F32toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.x < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_a.x++;\n\
        coord_b.y++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_b.y = get_global_id(1);\n\
    coord_b.z = get_global_id(2);\n\
    write_imagef(output, coord_b, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.zy);\n\
        tempB0 = read_imagef(inputB, coord.zx);\n\
        coord.z++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32F32toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, get_global_id(0), (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.x < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_a.x++;\n\
        coord_b.x++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_a.x = get_global_id(0);\n\
    coord_a.z = get_global_id(2);\n\
    write_imagef(output, coord_a, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32I8toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.zy);\n\
        tempB0 = convert_float4(read_imagei(inputB, coord.zx));\n\
        coord.z++;\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32I8toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, get_global_id(0), (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.x < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = convert_float4(read_imagei(inputB, coord_b));\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b;\n\
        coord_a.x++;\n\
        coord_b.x++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_a.x = get_global_id(0);\n\
    coord_a.z = get_global_id(2);\n\
    write_imagef(output, coord_a, sum);\n\
}\n\
\n\
#define GEMM_2D(name, dst_type, read_image_type, convert_type, write_image_type) \\\n\
__kernel void gemm_##name##_2D( \\\n\
    __read_only image2d_t   inputA, \\\n\
    __read_only image2d_t   inputB, \\\n\
    __write_only image2d_t  output, \\\n\
    int M, \\\n\
    int K, \\\n\
    int N, \\\n\
    int ac2zero, \\\n\
    int bc2zero, \\\n\
    float scale_a, \\\n\
    float zp_a, \\\n\
    float scale_b, \\\n\
    float zp_b, \\\n\
    float scale_out, \\\n\
    float zp_out \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    float4 sum = (float4)(0); \\\n\
    dst_type dst; \\\n\
\\\n\
    for(; coord.z < K;) \\\n\
    { \\\n\
        float4 tempA0; \\\n\
        float4 tempB0; \\\n\
        tempA0 = convert_float4(read_image_type(inputA, coord.zy)); \\\n\
        tempB0 = convert_float4(read_image_type(inputB, coord.xz)); \\\n\
        tempA0.x = (tempA0.x - zp_a) * scale_a; \\\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b; \\\n\
        coord.z++; \\\n\
        sum = sum + tempA0 * tempB0; \\\n\
    } \\\n\
    sum.x = sum.x * scale_out + zp_out; \\\n\
    dst = convert_type(sum); \\\n\
 \\\n\
    write_image_type(output, coord.xy, dst); \\\n\
}\n\
GEMM_2D(I8I8toI8,int4,read_imagei,convert_int4,write_imagei);\n\
GEMM_2D(U8U8toU8,uint4,read_imageui,convert_uint4,write_imageui);\n\
GEMM_2D(U8U8toF32,float4,read_imageui,convert_float4,write_imagef);\n\
\n\
\n\
#define GEMM_3D(name, dst_type, read_image_type, convert_type, write_image_type) \\\n\
__kernel void gemm_##name##_3D( \\\n\
    __read_only image2d_array_t   inputA, \\\n\
    __read_only image2d_array_t   inputB, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int M, \\\n\
    int K, \\\n\
    int N, \\\n\
    int ac2zero, \\\n\
    int bc2zero, \\\n\
    float scale_a, \\\n\
    float zp_a, \\\n\
    float scale_b, \\\n\
    float zp_b, \\\n\
    float scale_out, \\\n\
    float zp_out \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    float4 sum = (float4)(0); \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(; coord_a.x < K;) \\\n\
    { \\\n\
        float4 tempA0; \\\n\
        float4 tempB0; \\\n\
 \\\n\
        tempA0 = convert_float4(read_image_type(inputA, coord_a)); \\\n\
        tempB0 = convert_float4(read_image_type(inputB, coord_b)); \\\n\
        tempA0.x = (tempA0.x - zp_a) * scale_a; \\\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b; \\\n\
 \\\n\
        coord_a.x++; \\\n\
        coord_b.y++; \\\n\
 \\\n\
        sum = sum + tempA0 * tempB0; \\\n\
    } \\\n\
    sum.x = sum.x * scale_out + zp_out; \\\n\
    dst = convert_type(sum); \\\n\
 \\\n\
    coord_b.y = get_global_id(1); \\\n\
    coord_b.z = get_global_id(2); \\\n\
    write_image_type(output, coord_b, dst); \\\n\
}\n\
GEMM_3D(I8I8toI8,int4,read_imagei,convert_int4,write_imagei);\n\
GEMM_3D(U8U8toU8,uint4,read_imageui,convert_uint4,write_imageui);\n\
GEMM_3D(U8U8toF32,float4,read_imageui,convert_float4,write_imagef);\n\
\n\
#define GEMM_TRANSB_2D(name, dst_type, read_image_type, convert_type, write_image_type) \\\n\
__kernel void gemm_transb_##name##_2D( \\\n\
    __read_only image2d_t   inputA, \\\n\
    __read_only image2d_t   inputB, \\\n\
    __write_only image2d_t  output, \\\n\
    int M, \\\n\
    int K, \\\n\
    int N, \\\n\
    int ac2zero, \\\n\
    int bc2zero, \\\n\
    float scale_a, \\\n\
    float zp_a, \\\n\
    float scale_b, \\\n\
    float zp_b, \\\n\
    float scale_out, \\\n\
    float zp_out \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    float4 sum = (float4)(0); \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(; coord.z < K;) \\\n\
    { \\\n\
        float4 tempA0; \\\n\
        float4 tempB0; \\\n\
 \\\n\
        tempA0 = convert_float4(read_image_type(inputA, coord.zy)); \\\n\
        tempB0 = convert_float4(read_image_type(inputB, coord.zx)); \\\n\
        tempA0.x = (tempA0.x - zp_a) * scale_a; \\\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b; \\\n\
        coord.z++; \\\n\
 \\\n\
        sum = sum + tempA0 * tempB0; \\\n\
    } \\\n\
    sum.x = sum.x * scale_out + zp_out; \\\n\
    dst = convert_type(sum); \\\n\
 \\\n\
    write_image_type(output, coord.xy, dst); \\\n\
}\n\
GEMM_TRANSB_2D(I8I8toI8,int4,read_imagei,convert_int4,write_imagei);\n\
GEMM_TRANSB_2D(U8U8toU8,uint4,read_imageui,convert_uint4,write_imageui);\n\
GEMM_TRANSB_2D(U8U8toF32,float4,read_imageui,convert_float4,write_imagef);\n\
\n\
\n\
#define GEMM_TRANSB_3D(name, dst_type, read_image_type, convert_type, write_image_type) \\\n\
__kernel void gemm_transb_##name##_3D( \\\n\
    __read_only image2d_array_t   inputA, \\\n\
    __read_only image2d_array_t   inputB, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int M, \\\n\
    int K, \\\n\
    int N, \\\n\
    int ac2zero, \\\n\
    int bc2zero, \\\n\
    float scale_a, \\\n\
    float zp_a, \\\n\
    float scale_b, \\\n\
    float zp_b, \\\n\
    float scale_out, \\\n\
    float zp_out \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(0, get_global_id(0), (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    float4 sum = (float4)(0); \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(; coord_a.x < K;) \\\n\
    { \\\n\
        float4 tempA0; \\\n\
        float4 tempB0; \\\n\
 \\\n\
        tempA0 = convert_float4(read_image_type(inputA, coord_a)); \\\n\
        tempB0 = convert_float4(read_image_type(inputB, coord_b)); \\\n\
        tempA0.x = (tempA0.x - zp_a) * scale_a; \\\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b; \\\n\
        coord_a.x++; \\\n\
        coord_b.x++; \\\n\
 \\\n\
        sum = sum + tempA0 * tempB0; \\\n\
    } \\\n\
    sum.x = sum.x * scale_out + zp_out; \\\n\
    dst = convert_type(sum); \\\n\
 \\\n\
    coord_a.x = get_global_id(0); \\\n\
    coord_a.z = get_global_id(2); \\\n\
    write_image_type(output, coord_a, dst); \\\n\
}\n\
GEMM_TRANSB_3D(I8I8toI8,int4,read_imagei,convert_int4,write_imagei);\n\
GEMM_TRANSB_3D(U8U8toU8,uint4,read_imageui,convert_uint4,write_imageui);\n\
GEMM_TRANSB_3D(U8U8toF32,float4,read_imageui,convert_float4,write_imagef);\n\
\n\
"; /* end of matrixmul_cl*/

static const char matrixmul_4x_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
\n\
__kernel void gemm_4x_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int offset0 = get_global_id(0) * K;\n\
    int offset1 = offset0 + K;\n\
    int offset2 = offset1 + K;\n\
    int offset3 = offset2 + K;\n\
    int out_offset = get_global_id(0);\n\
    int z = 0;\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
\n\
    Image in0_tensor = create_image_from_image2d(inputA, 4);\n\
    __global float* in0_ptr = (__global float*)in0_tensor.ptr;\n\
    __global float* in0_ptr0 = in0_ptr + offset0;\n\
    __global float* in0_ptr1 = in0_ptr + offset1;\n\
    __global float* in0_ptr2 = in0_ptr + offset2;\n\
    __global float* in0_ptr3 = in0_ptr + offset3;\n\
\n\
    Image in1_tensor = create_image_from_image2d(inputB, 4);\n\
    __global float* in1_ptr = (__global float*)in1_tensor.ptr;\n\
\n\
    Image o_tensor = create_image_from_image2d(output, 4);\n\
    __global float* output_ptr = (__global float*)o_tensor.ptr + out_offset;\n\
\n\
    int step = K >> 2;\n\
    for(z = 0; z < step; z++)\n\
    {\n\
        float4 tempA0, tempA1, tempA2, tempA3;\n\
        float4 tempB0;\n\
\n\
        tempB0 = vload4(z, in1_ptr);\n\
        tempA0 = vload4(z, in0_ptr0);\n\
        tempA1 = vload4(z, in0_ptr1);\n\
        tempA2 = vload4(z, in0_ptr2);\n\
        tempA3 = vload4(z, in0_ptr3);\n\
\n\
        sum.x += dot(tempA0, tempB0);\n\
        sum.y += dot(tempA1, tempB0);\n\
        sum.z += dot(tempA2, tempB0);\n\
        sum.w += dot(tempA3, tempB0);\n\
    }\n\
\n\
    vstore4(sum, 0, output_ptr);\n\
\n\
}\n\
\n\
__kernel void gemm_4x_transa_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int offset0 = get_global_id(0);\n\
    int offset1 = M << 2;\n\
\n\
    int z = 0;\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
\n\
    Image in0_tensor = create_image_from_image2d(inputA, 4);\n\
    __global float* in0_ptr0 = (__global float*)in0_tensor.ptr + offset0;\n\
    __global float* in0_ptr1 = in0_ptr0 + M;\n\
    __global float* in0_ptr2 = in0_ptr1 + M;\n\
    __global float* in0_ptr3 = in0_ptr2 + M;\n\
\n\
    Image in1_tensor = create_image_from_image2d(inputB, 4);\n\
    __global float* in1_ptr = (__global float*)in1_tensor.ptr;\n\
\n\
    Image o_tensor = create_image_from_image2d(output, 4);\n\
    __global float* output_ptr = (__global float*)o_tensor.ptr + offset0;\n\
\n\
    int step = K >> 2;\n\
    for(z = 0; z < step; z++)\n\
    {\n\
        float4 tempA0, tempA1, tempA2, tempA3;\n\
        float4 tempB0;\n\
\n\
        tempB0 = vload4(z, in1_ptr);\n\
        tempA0 = vload4(0, in0_ptr0);\n\
        tempA1 = vload4(0, in0_ptr1);\n\
        tempA2 = vload4(0, in0_ptr2);\n\
        tempA3 = vload4(0, in0_ptr3);\n\
\n\
        sum += tempA0 * tempB0.x;\n\
        sum += tempA1 * tempB0.y;\n\
        sum += tempA2 * tempB0.z;\n\
        sum += tempA3 * tempB0.w;\n\
\n\
        in0_ptr0 = in0_ptr0 + offset1;\n\
        in0_ptr1 = in0_ptr1 + offset1;\n\
        in0_ptr2 = in0_ptr2 + offset1;\n\
        in0_ptr3 = in0_ptr3 + offset1;\n\
\n\
    }\n\
\n\
    vstore4(sum, 0, output_ptr);\n\
\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(1, 64, 1)))\n\
    void gemm_4x_transa_local_F32F32toF32_2D(\n\
    __read_only  image2d_t inputA,\n\
    __read_only  image2d_t inputB,\n\
    __write_only image2d_t output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int offset0 = get_global_id(0);\n\
    int lid = get_local_id(1);\n\
\n\
    int stride = 0;\n\
\n\
    int z = 0;\n\
    int offset1 = M << 2;\n\
    int step = K >> 8;\n\
    int lid2 = lid * 4 * step;\n\
\n\
    Image in0_tensor = create_image_from_image2d(inputA, 4);\n\
    __global float* in0_ptr0 = (__global float*)in0_tensor.ptr + offset0 + lid2 * M;\n\
    __global float* in0_ptr1 = in0_ptr0 + M;\n\
    __global float* in0_ptr2 = in0_ptr1 + M;\n\
    __global float* in0_ptr3 = in0_ptr2 + M;\n\
\n\
    Image in1_tensor = create_image_from_image2d(inputB, 4);\n\
    __global float* in1_ptr = (__global float*)in1_tensor.ptr + lid2;\n\
\n\
    Image o_tensor = create_image_from_image2d(output, 4);\n\
    __global float* output_ptr = (__global float*)o_tensor.ptr + offset0;\n\
\n\
    __local float4 sum_vec4_0[64];\n\
    __local float4 sum_vec4_1[64];\n\
    __local float4 sum_vec4_2[64];\n\
    __local float4 sum_vec4_3[64];\n\
\n\
    float4 sum0 = (float4)(0.0, 0.0, 0.0, 0.0);\n\
    float4 sum1 = (float4)(0.0, 0.0, 0.0, 0.0);\n\
    float4 sum2 = (float4)(0.0, 0.0, 0.0, 0.0);\n\
    float4 sum3 = (float4)(0.0, 0.0, 0.0, 0.0);\n\
\n\
    float4 tempA0, tempA1, tempA2, tempA3;\n\
    float4 tempA4, tempA5, tempA6, tempA7;\n\
    float4 tempB0;\n\
\n\
    for(z = 0; z < step; z++)\n\
    {\n\
        tempB0 = vload4(z, in1_ptr);\n\
        tempA0 = vload4(0, in0_ptr0);\n\
        tempA1 = vload4(0, in0_ptr1);\n\
        tempA2 = vload4(0, in0_ptr2);\n\
        tempA3 = vload4(0, in0_ptr3);\n\
        tempA4 = vload4(1, in0_ptr0);\n\
        tempA5 = vload4(1, in0_ptr1);\n\
        tempA6 = vload4(1, in0_ptr2);\n\
        tempA7 = vload4(1, in0_ptr3);\n\
\n\
        sum0 = sum0 + tempA0 * tempB0.x;\n\
        sum0 = sum0 + tempA1 * tempB0.y;\n\
        sum0 = sum0 + tempA2 * tempB0.z;\n\
        sum0 = sum0 + tempA3 * tempB0.w;\n\
        sum1 = sum1 + tempA4 * tempB0.x;\n\
        sum1 = sum1 + tempA5 * tempB0.y;\n\
        sum1 = sum1 + tempA6 * tempB0.z;\n\
        sum1 = sum1 + tempA7 * tempB0.w;\n\
\n\
        tempA0 = vload4(2, in0_ptr0);\n\
        tempA1 = vload4(2, in0_ptr1);\n\
        tempA2 = vload4(2, in0_ptr2);\n\
        tempA3 = vload4(2, in0_ptr3);\n\
        tempA4 = vload4(3, in0_ptr0);\n\
        tempA5 = vload4(3, in0_ptr1);\n\
        tempA6 = vload4(3, in0_ptr2);\n\
        tempA7 = vload4(3, in0_ptr3);\n\
\n\
        in0_ptr0 = in0_ptr0 + offset1;\n\
        in0_ptr1 = in0_ptr1 + offset1;\n\
        in0_ptr2 = in0_ptr2 + offset1;\n\
        in0_ptr3 = in0_ptr3 + offset1;\n\
\n\
        sum2 = sum2 + tempA0 * tempB0.x;\n\
        sum2 = sum2 + tempA1 * tempB0.y;\n\
        sum2 = sum2 + tempA2 * tempB0.z;\n\
        sum2 = sum2 + tempA3 * tempB0.w;\n\
        sum3 = sum3 + tempA4 * tempB0.x;\n\
        sum3 = sum3 + tempA5 * tempB0.y;\n\
        sum3 = sum3 + tempA6 * tempB0.z;\n\
        sum3 = sum3 + tempA7 * tempB0.w;\n\
    }\n\
    sum_vec4_0[lid] = sum0;\n\
    sum_vec4_1[lid] = sum1;\n\
    sum_vec4_2[lid] = sum2;\n\
    sum_vec4_3[lid] = sum3;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    for (stride = 32; stride > 0; stride >>= 1)\n\
    {\n\
        if (lid < stride)\n\
        {\n\
            sum_vec4_0[lid] += sum_vec4_0[lid + stride];\n\
            sum_vec4_1[lid] += sum_vec4_1[lid + stride];\n\
            sum_vec4_2[lid] += sum_vec4_2[lid + stride];\n\
            sum_vec4_3[lid] += sum_vec4_3[lid + stride];\n\
        }\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
    }\n\
\n\
    if (lid == 0)\n\
    {\n\
        sum0 = sum_vec4_0[0];\n\
        sum1 = sum_vec4_1[0];\n\
        sum2 = sum_vec4_2[0];\n\
        sum3 = sum_vec4_3[0];\n\
        vstore4(sum0, 0, output_ptr);\n\
        vstore4(sum1, 1, output_ptr);\n\
        vstore4(sum2, 2, output_ptr);\n\
        vstore4(sum3, 3, output_ptr);\n\
    }\n\
}\n\
\n\
"; /* end of matrixmul_4x_cl*/

static const char matrixmul_cross_cl[] = "__kernel void gemm_F32F32toF32_merge(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out,\n\
    int outer)\n\
{\n\
    for(int i = 0; i < outer; i++)\n\
    {\n\
        int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? i : get_global_id(2)), 0);\n\
        int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? i : get_global_id(2)), 0);\n\
\n\
        float4 sum = (float4)(0);\n\
\n\
        for(; coord_a.x < K;)\n\
        {\n\
            float4 tempA0;\n\
            float4 tempB0;\n\
\n\
            tempA0 = read_imagef(inputA, coord_a);\n\
            tempB0 = read_imagef(inputB, coord_b);\n\
            coord_a.x++;\n\
            coord_b.y++;\n\
\n\
            sum = sum + tempA0 * tempB0;\n\
        }\n\
\n\
        coord_b.y = get_global_id(1);\n\
        coord_b.z = get_global_id(2) + i * get_global_size(2);\n\
        write_imagef(output, coord_b, sum);\n\
    }\n\
}\n\
\n\
#define GEMM_MERGE(name, dst_type, read_image_type, convert_type, write_image_type) \\\n\
__kernel void gemm_##name##_merge( \\\n\
    __read_only image2d_array_t   inputA, \\\n\
    __read_only image2d_array_t   inputB, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int M, \\\n\
    int K, \\\n\
    int N, \\\n\
    int ac2zero, \\\n\
    int bc2zero, \\\n\
    float scale_a, \\\n\
    float zp_a, \\\n\
    float scale_b, \\\n\
    float zp_b, \\\n\
    float scale_out, \\\n\
    float zp_out, \\\n\
    int outer) \\\n\
{ \\\n\
    for(int i = 0; i < outer; i++) \\\n\
    { \\\n\
        int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? i : get_global_id(2)), 0); \\\n\
        int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? i : get_global_id(2)), 0); \\\n\
        float4 sum = (float4)(0); \\\n\
        dst_type dst; \\\n\
     \\\n\
        for(; coord_a.x < K;) \\\n\
        { \\\n\
            float4 tempA0; \\\n\
            float4 tempB0; \\\n\
     \\\n\
            tempA0 = convert_float4(read_image_type(inputA, coord_a)); \\\n\
            tempB0 = convert_float4(read_image_type(inputB, coord_b)); \\\n\
            tempA0.x = (tempA0.x - zp_a) * scale_a; \\\n\
            tempB0.x = (tempB0.x - zp_b) * scale_b; \\\n\
     \\\n\
            coord_a.x++; \\\n\
            coord_b.y++; \\\n\
     \\\n\
            sum = sum + tempA0 * tempB0; \\\n\
        } \\\n\
        sum.x = sum.x * scale_out + zp_out; \\\n\
        dst = convert_type(sum); \\\n\
     \\\n\
        coord_b.y = get_global_id(1); \\\n\
        coord_b.z = get_global_id(2) + i * get_global_size(2); \\\n\
        write_image_type(output, coord_b, dst); \\\n\
    } \\\n\
}\n\
GEMM_MERGE(I8I8toI8,int4,read_imagei,convert_int4,write_imagei);\n\
GEMM_MERGE(U8U8toU8,uint4,read_imageui,convert_uint4,write_imageui);\n\
GEMM_MERGE(U8U8toF32,float4,read_imageui,convert_float4,write_imagef);\n\
\n\
"; /* end of matrixmul_cross_cl*/

static const char matrixmul_transA_cl[] = "__kernel void gemm_transa_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.yz);\n\
        tempB0 = read_imagef(inputB, coord.xz);\n\
        coord.z++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_transa_F32F32toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(gidx, 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.y < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_a.y++;\n\
        coord_b.y++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    write_imagef(output, coord_b, sum);\n\
}\n\
\n\
#define GEMM_TRANSA_2D(name, dst_type, read_image_type, convert_type, write_image_type) \\\n\
__kernel void gemm_transa_##name##_2D( \\\n\
    __read_only image2d_t   inputA, \\\n\
    __read_only image2d_t   inputB, \\\n\
    __write_only image2d_t  output, \\\n\
    int M, \\\n\
    int K, \\\n\
    int N, \\\n\
    int ac2zero, \\\n\
    int bc2zero, \\\n\
    float scale_a, \\\n\
    float zp_a, \\\n\
    float scale_b, \\\n\
    float zp_b, \\\n\
    float scale_out, \\\n\
    float zp_out \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    float4 sum = (float4)(0); \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(; coord.z < K;) \\\n\
    { \\\n\
        float4 tempA0; \\\n\
        float4 tempB0; \\\n\
 \\\n\
        tempA0 = convert_float4(read_image_type(inputA, coord.yz)); \\\n\
        tempB0 = convert_float4(read_image_type(inputB, coord.xz)); \\\n\
        tempA0.x = (tempA0.x - zp_a) * scale_a; \\\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b; \\\n\
        coord.z++; \\\n\
 \\\n\
        sum = sum + tempA0 * tempB0; \\\n\
    } \\\n\
    sum.x = sum.x * scale_out + zp_out; \\\n\
    dst = convert_type(sum); \\\n\
 \\\n\
    write_image_type(output, coord.xy, dst); \\\n\
}\n\
GEMM_TRANSA_2D(I8I8toI8,int4,read_imagei,convert_int4,write_imagei);\n\
GEMM_TRANSA_2D(U8U8toU8,uint4,read_imageui,convert_uint4,write_imageui);\n\
GEMM_TRANSA_2D(U8U8toF32,float4,read_imageui,convert_float4,write_imagef);\n\
\n\
#define GEMM_TRANSA_3D(name, dst_type, read_image_type, convert_type, write_image_type) \\\n\
__kernel void gemm_transa_##name##_3D( \\\n\
    __read_only image2d_array_t   inputA, \\\n\
    __read_only image2d_array_t   inputB, \\\n\
    __write_only image2d_array_t  output, \\\n\
    int M, \\\n\
    int K, \\\n\
    int N, \\\n\
    int ac2zero, \\\n\
    int bc2zero, \\\n\
    float scale_a, \\\n\
    float zp_a, \\\n\
    float scale_b, \\\n\
    float zp_b, \\\n\
    float scale_out, \\\n\
    float zp_out \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(gidx, 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    float4 sum = (float4)(0); \\\n\
    dst_type dst; \\\n\
 \\\n\
    for(; coord_a.y < K;) \\\n\
    { \\\n\
        float4 tempA0; \\\n\
        float4 tempB0; \\\n\
 \\\n\
        tempA0 = convert_float4(read_image_type(inputA, coord_a)); \\\n\
        tempB0 = convert_float4(read_image_type(inputB, coord_b)); \\\n\
        tempA0.x = (tempA0.x - zp_a) * scale_a; \\\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b; \\\n\
        coord_a.y++; \\\n\
        coord_b.y++; \\\n\
 \\\n\
        sum = sum + tempA0 * tempB0; \\\n\
    } \\\n\
    sum.x = sum.x * scale_out + zp_out; \\\n\
    dst = convert_type(sum); \\\n\
 \\\n\
    coord_b.y = gidy; \\\n\
    coord_b.z = get_global_id(2); \\\n\
    write_image_type(output, coord_b, dst); \\\n\
}\n\
GEMM_TRANSA_3D(I8I8toI8,int4,read_imagei,convert_int4,write_imagei);\n\
GEMM_TRANSA_3D(U8U8toU8,uint4,read_imageui,convert_uint4,write_imageui);\n\
GEMM_TRANSA_3D(U8U8toF32,float4,read_imageui,convert_float4,write_imagef);\n\
"; /* end of matrixmul_transA_cl*/

static const char maximum_cl[] = "__kernel void maximum_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4 dst = src0 > src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 dst = src0 > src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 src0;\n\
    uint4 src1;\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 > data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    uint4 src0 = read_imageui(input0, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 > data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
\n\
__kernel void maximum_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 > data1 ? data0 : data1;\n\
    int4 dst = convert_int4_rte(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    int4 src0 = read_imagei(input0, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 > data1 ? data0 : data1;\n\
    int4 dst = convert_int4_rte(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of maximum_cl*/

static const char maxpool_cl[] = "#define VSI_FLOAT32_MIN     (-3.40E+38)\n\
\n\
#define MAXPOOL_QINT(in_name, out_name, src_type, dst_type, max_val, read_func, write_func, conv_func) \\\n\
__kernel void maxpool_##in_name##to##out_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              width, \\\n\
                 int              height, \\\n\
                 int              stride_x, \\\n\
                 int              stride_y, \\\n\
                 int              pad_x, \\\n\
                 int              pad_y, \\\n\
                 int              kernel_dia_x, \\\n\
                 int              kernel_dia_y, \\\n\
                 int              dilation_x, \\\n\
                 int              dilation_y, \\\n\
                 float            inout_scale, \\\n\
                 float            inout_tail) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y); \\\n\
    int4 coord_in = coord_out; \\\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y); \\\n\
 \\\n\
    for(; pos_start.x < 0;) \\\n\
    { \\\n\
        pos_start.x += dilation_x; \\\n\
    } \\\n\
    for(; pos_start.y < 0;) \\\n\
    { \\\n\
        pos_start.y += dilation_y; \\\n\
    } \\\n\
 \\\n\
    pos_end = min(pos_end, (int2)(width, height)); \\\n\
 \\\n\
    src_type src0, maxVal; \\\n\
    maxVal.x = max_val; \\\n\
 \\\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y) \\\n\
    { \\\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;) \\\n\
        { \\\n\
            src0 = read_func(input, coord_in); \\\n\
            coord_in.x += dilation_x; \\\n\
            maxVal.x = src0.x > maxVal.x ? src0.x : maxVal.x; \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    float4 fValTmp; \\\n\
    fValTmp.x = maxVal.x * inout_scale + inout_tail; \\\n\
    dst_type dst = conv_func(fValTmp); \\\n\
    write_func(output, coord_out, dst.xxxx); \\\n\
}\n\
MAXPOOL_QINT(U32, U32, uint4, uint4, 0, read_imageui, write_imageui, convert_uint4_rte)\n\
MAXPOOL_QINT(I32, I32, int4, int4, -2147483648, read_imagei, write_imagei, convert_int4_rte)\n\
\n\
__kernel void maxpool_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 int              width,\n\
                 int              height,\n\
                 int              stride_x,\n\
                 int              stride_y,\n\
                 int              pad_x,\n\
                 int              pad_y,\n\
                 int              kernel_dia_x,\n\
                 int              kernel_dia_y,\n\
                 int              dilation_x,\n\
                 int              dilation_y,\n\
                 float            inout_scale,\n\
                 float            inout_tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y);\n\
    int4 coord_in = coord_out;\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y);\n\
\n\
    for(; pos_start.x < 0;)\n\
    {\n\
        pos_start.x += dilation_x;\n\
    }\n\
    for(; pos_start.y < 0;)\n\
    {\n\
        pos_start.y += dilation_y;\n\
    }\n\
\n\
    pos_end = min(pos_end, (int2)(width, height));\n\
\n\
    float4 src0, maxVal;\n\
    maxVal.x = VSI_FLOAT32_MIN;\n\
\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y)\n\
    {\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;)\n\
        {\n\
            src0 = read_imagef(input, coord_in);\n\
            coord_in.x += dilation_x;\n\
            maxVal.x = src0.x > maxVal.x ? src0.x : maxVal.x;\n\
        }\n\
    }\n\
\n\
    write_imagef(output, coord_out, maxVal.xxxx);\n\
}\n\
\n\
__kernel void maxpool_U32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 int              width,\n\
                 int              height,\n\
                 int              stride_x,\n\
                 int              stride_y,\n\
                 int              pad_x,\n\
                 int              pad_y,\n\
                 int              kernel_dia_x,\n\
                 int              kernel_dia_y,\n\
                 int              dilation_x,\n\
                 int              dilation_y,\n\
                 float            inout_scale,\n\
                 float            inout_tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y);\n\
    int4 coord_in = coord_out;\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y);\n\
\n\
    for(; pos_start.x < 0;)\n\
    {\n\
        pos_start.x += dilation_x;\n\
    }\n\
    for(; pos_start.y < 0;)\n\
    {\n\
        pos_start.y += dilation_y;\n\
    }\n\
\n\
    pos_end = min(pos_end, (int2)(width, height));\n\
\n\
    uint4 src0, maxVal;\n\
    maxVal.x = 0;\n\
\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y)\n\
    {\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;)\n\
        {\n\
            src0 = read_imageui(input, coord_in);\n\
            coord_in.x += dilation_x;\n\
            maxVal.x = src0.x > maxVal.x ? src0.x : maxVal.x;\n\
        }\n\
    }\n\
\n\
    float4 dst;\n\
    dst.x = maxVal.x * inout_scale + inout_tail;\n\
\n\
    write_imagef(output, coord_out, dst.xxxx);\n\
}\n\
\n\
__kernel void maxpool_F32toU32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 int              width,\n\
                 int              height,\n\
                 int              stride_x,\n\
                 int              stride_y,\n\
                 int              pad_x,\n\
                 int              pad_y,\n\
                 int              kernel_dia_x,\n\
                 int              kernel_dia_y,\n\
                 int              dilation_x,\n\
                 int              dilation_y,\n\
                 float            inout_scale,\n\
                 float            inout_tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int2 pos_start = coord_out.xy * (int2)(stride_x, stride_y) - (int2)(pad_x, pad_y);\n\
    int4 coord_in = coord_out;\n\
    int2 pos_end = pos_start + (int2)(kernel_dia_x, kernel_dia_y);\n\
\n\
    for(; pos_start.x < 0;)\n\
    {\n\
        pos_start.x += dilation_x;\n\
    }\n\
    for(; pos_start.y < 0;)\n\
    {\n\
        pos_start.y += dilation_y;\n\
    }\n\
\n\
    pos_end = min(pos_end, (int2)(width, height));\n\
\n\
    float4 src0, maxVal;\n\
    maxVal.x = VSI_FLOAT32_MIN;\n\
\n\
    for(coord_in.y = pos_start.y; coord_in.y < pos_end.y; coord_in.y += dilation_y)\n\
    {\n\
        for(coord_in.x = pos_start.x; coord_in.x < pos_end.x;)\n\
        {\n\
            src0 = read_imagef(input, coord_in);\n\
            coord_in.x += dilation_x;\n\
            maxVal.x = src0.x > maxVal.x ? src0.x : maxVal.x;\n\
        }\n\
    }\n\
\n\
    uint4 dst;\n\
    dst.x = convert_uint_rte(maxVal.x * inout_scale + inout_tail);\n\
\n\
    write_imageui(output, coord_out, dst.xxxx);\n\
}\n\
"; /* end of maxpool_cl*/

static const char maxpoolwithargmax_cl[] = "#define FP32_MIN   -3.4e38\n\
#define I32_MIN    -2147483647\n\
\n\
__kernel void maxpoolwithargmax_F32toF32_I32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord_out = (int4)(gidx, gidy, get_global_id(2), 0);\n\
    int4 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    float value_max = FP32_MIN;\n\
    float4 dst = (float4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            float4 data = read_imagef(input, coord_in);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in.xy;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width + get_global_id(2) * width * height;\n\
    dst.x = value_max;\n\
    write_imagef(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
\n\
__kernel void maxpoolwithargmax_BF16toBF16_I32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord_out = (int4)(gidx, gidy, get_global_id(2), 0);\n\
    int4 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    float value_max = FP32_MIN;\n\
    uint4 dst = (uint4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            uint4 src = read_imageui(input, coord_in);\n\
            src = src << 16;\n\
            float4 data;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in.xy;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width + get_global_id(2) * width * height;\n\
    _viv_asm(COPY, dst, value_max, 4);\n\
    dst.x = dst.x >> 16;\n\
    write_imageui(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
\n\
__kernel void maxpoolwithargmax_U32toU32_I32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord_out = (int4)(gidx, gidy, get_global_id(2), 0);\n\
    int4 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    uint value_max = 0;\n\
    uint4 dst = (uint4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            uint4 data = read_imageui(input, coord_in);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in.xy;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width + get_global_id(2) * width * height;\n\
    dst.x = convert_uint(convert_float(value_max) * scale + tail);\n\
    write_imageui(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
\n\
__kernel void maxpoolwithargmax_I32toI32_I32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord_out = (int4)(gidx, gidy, get_global_id(2), 0);\n\
    int4 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    int value_max = I32_MIN;\n\
    int4 dst = (int4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            int4 data = read_imagei(input, coord_in);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in.xy;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width + get_global_id(2) * width * height;\n\
    dst.x = convert_int(convert_float(value_max) * scale + tail);\n\
    write_imagei(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
"; /* end of maxpoolwithargmax_cl*/

static const char maxpoolwithargmax_2d_cl[] = "#define FP32_MIN   -3.4e38\n\
#define I32_MIN    -2147483647\n\
\n\
__kernel void maxpoolwithargmax_F32toF32_I32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    __write_only image2d_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    int2 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    float value_max = FP32_MIN;\n\
    float4 dst = (float4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            float4 data = read_imagef(input, coord_in);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width;\n\
    dst.x = value_max;\n\
    write_imagef(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
\n\
__kernel void maxpoolwithargmax_BF16toBF16_I32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    __write_only image2d_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    int2 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    float value_max = FP32_MIN;\n\
    uint4 dst = (uint4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            uint4 src = read_imageui(input, coord_in);\n\
            src = src << 16;\n\
            float4 data;\n\
            _viv_asm(COPY, data, src, 16);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width;\n\
    _viv_asm(COPY, dst, value_max, 4);\n\
    dst.x = dst.x >> 16;\n\
    write_imageui(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
\n\
__kernel void maxpoolwithargmax_U32toU32_I32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    __write_only image2d_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    int2 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    uint value_max = 0;\n\
    uint4 dst = (uint4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            uint4 data = read_imageui(input, coord_in);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width;\n\
    dst.x = convert_uint(convert_float(value_max) * scale + tail);\n\
    write_imageui(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
\n\
__kernel void maxpoolwithargmax_I32toI32_I32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    __write_only image2d_t  argmax,\n\
    int ksize_x, int ksize_y, int stride_x, int stride_y,\n\
    int pad_left, int pad_top, int width, int height,\n\
    float scale, float tail)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    int2 coord_in  = coord_out;\n\
\n\
    int hstart = gidy * stride_y - pad_top;\n\
    int wstart = gidx * stride_x - pad_left;\n\
    int hend = min(hstart + ksize_y, height);\n\
    int wend = min(wstart + ksize_x, width);\n\
    int h, w;\n\
    int4 index_max = (int4)(0);\n\
    int value_max = I32_MIN;\n\
    int4 dst = (int4)(0);\n\
\n\
    hstart = max(hstart, 0);\n\
    wstart = max(wstart, 0);\n\
    int2 coord_max = (int2)(wstart, hstart);\n\
    for (h = hstart; h < hend; ++ h)\n\
    {\n\
        for (w = wstart; w < wend; ++ w)\n\
        {\n\
            coord_in.xy = (int2)(w, h);\n\
            int4 data = read_imagei(input, coord_in);\n\
\n\
            if (data.x > value_max)\n\
            {\n\
                value_max = data.x;\n\
                coord_max = coord_in;\n\
            }\n\
        }\n\
    }\n\
\n\
    index_max.x = coord_max.x + coord_max.y * width;\n\
    dst.x = convert_int(convert_float(value_max) * scale + tail);\n\
    write_imagei(output, coord_out, dst);\n\
    write_imagei(argmax, coord_out, index_max);\n\
}\n\
"; /* end of maxpoolwithargmax_2d_cl*/

static const char maxunpool_cl[] = "\n\
#define MAXUNPOOL(name, read_type, read_image_type, write_type, convert_type, writeimage_type) \\\n\
__kernel void maxunpool_##name( \\\n\
    __read_only  image2d_array_t  input0, \\\n\
    __read_only  image2d_array_t  input1, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              width_nopad, \\\n\
                 int              height_nopad, \\\n\
                 int              width_in, \\\n\
                 int              height_in, \\\n\
                 int              batch, \\\n\
                 int              pad_left, \\\n\
                 int              pad_top, \\\n\
                 float            inputScale, \\\n\
                 float            inputTail, \\\n\
                 float            outputScale, \\\n\
                 float            outputTail \\\n\
    ) \\\n\
{ \\\n\
    uint gidx = get_global_id(0); \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    int gidx_in, gidy_in, gidz_in; \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    write_type dst = (write_type)(0); \\\n\
    float4 dst_temp = (float4)(0); \\\n\
    int i,j,k; \\\n\
    if (gidx < pad_left || gidx >= width_nopad + pad_left || \\\n\
        gidy < pad_top || gidy >= height_nopad + pad_top) \\\n\
    { \\\n\
        dst_temp.x = outputTail; \\\n\
        dst = convert_type(dst_temp); \\\n\
        writeimage_type(output, coord_out, dst); \\\n\
        return; \\\n\
    } \\\n\
    gidx_in = gidx - pad_left; \\\n\
    gidy_in = gidy - pad_top; \\\n\
    gidz_in = gidz; \\\n\
    int index = gidz_in * height_nopad * width_nopad + gidy_in * width_nopad + gidx_in; \\\n\
    for (k = 0;k < batch;k++) \\\n\
    { \\\n\
        for (j = 0;j < height_in; j++) \\\n\
        { \\\n\
            for (i = 0;i < width_in; i++) \\\n\
            { \\\n\
                int index_useful = read_imagei(input1, (int4)(i,j,k,0)).x; \\\n\
                if (index_useful == index) \\\n\
                { \\\n\
                    read_type src = read_image_type(input0, (int4)(i,j,k,0)); \\\n\
                    dst_temp = convert_float4(src) * inputScale + inputTail; \\\n\
                    dst = convert_type(dst_temp * outputScale + outputTail); \\\n\
                    writeimage_type(output, coord_out, dst); \\\n\
                    return; \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    dst_temp.x = outputTail; \\\n\
    dst = convert_type(dst_temp); \\\n\
    writeimage_type(output, coord_out, dst); \\\n\
}\n\
MAXUNPOOL(F32toF32,float4,read_imagef,float4,convert_float4,write_imagef)\n\
MAXUNPOOL(F32toU32,float4,read_imagef,uint4, convert_uint4, write_imageui)\n\
MAXUNPOOL(F32toI32,float4,read_imagef,int4,  convert_int4,  write_imagei)\n\
\n\
MAXUNPOOL(U32toU32,uint4,read_imageui,uint4, convert_uint4, write_imageui)\n\
MAXUNPOOL(U32toF32,uint4,read_imageui,float4,convert_float4,write_imagef)\n\
MAXUNPOOL(U32toI32,uint4,read_imageui,int4,  convert_int4,  write_imagei)\n\
\n\
MAXUNPOOL(I32toU32,int4,read_imagei,uint4, convert_uint4, write_imageui)\n\
MAXUNPOOL(I32toF32,int4,read_imagei,float4,convert_float4,write_imagef)\n\
MAXUNPOOL(I32toI32,int4,read_imagei,int4,  convert_int4,  write_imagei)\n\
\n\
__kernel void maxunpool_BF16toBF16(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 int              width_nopad,\n\
                 int              height_nopad,\n\
                 int              width_in,\n\
                 int              height_in,\n\
                 int              batch,\n\
                 int              pad_left,\n\
                 int              pad_top,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
    )\n\
{\n\
    uint gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    uint gidz = get_global_id(2);\n\
    int gidx_in, gidy_in, gidz_in;\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    uint4 dst = (uint4)(0);\n\
    float4 dst_temp = (float4)(0);\n\
    int i,j,k;\n\
    if (gidx < pad_left || gidx >= width_nopad + pad_left ||\n\
        gidy < pad_top || gidy >= height_nopad + pad_top)\n\
    {\n\
        dst_temp.x = 0;\n\
        _viv_asm(COPY, dst, dst_temp, 16);\n\
        dst.x = dst.x >> 16;\n\
        write_imageui(output, coord_out, dst);\n\
        return;\n\
    }\n\
    gidx_in = gidx - pad_left;\n\
    gidy_in = gidy - pad_top;\n\
    gidz_in = gidz;\n\
    int index = gidz_in * height_nopad * width_nopad + gidy_in * width_nopad + gidx_in;\n\
    for (k = 0;k < batch;k++)\n\
    {\n\
        for (j = 0;j < height_in; j++)\n\
        {\n\
            for (i = 0;i < width_in; i++)\n\
            {\n\
                int index_useful = read_imagei(input1, (int4)(i,j,k,0)).x;\n\
                if (index_useful == index)\n\
                {\n\
                    uint4 src = read_imageui(input0, (int4)(i,j,k,0));\n\
                    write_imageui(output, coord_out, src);\n\
                    return;\n\
                }\n\
            }\n\
        }\n\
    }\n\
    dst_temp.x = 0;\n\
    _viv_asm(COPY, dst, dst_temp, 16);\n\
    dst.x = dst.x >> 16;\n\
    write_imageui(output, coord_out, dst);\n\
}"; /* end of maxunpool_cl*/

static const char minimum_cl[] = "__kernel void minimum_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4 dst = src0 < src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 dst = src0 < src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 src0;\n\
    uint4 src1;\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 < data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    uint4 src0 = read_imageui(input0, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 < data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
\n\
__kernel void minimum_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 < data1 ? data0 : data1;\n\
    int4 dst = convert_int4_rte(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    int4 src0 = read_imagei(input0, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 < data1 ? data0 : data1;\n\
    int4 dst = convert_int4_rte(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of minimum_cl*/

static const char mod_cl[] = "__kernel void mod_F32F32toF32\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 int              isfmod,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
    float4 dst  = fmod(src0, src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_F32F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 int       isfmod,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src0 = read_imagef(input, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
    float4 dst  = fmod(src0, src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 int              isfmod,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out;\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    int4 dst = convert_int4(out);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 int       isfmod,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src0 = read_imagei(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out;\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    int4 dst = convert_int4(out);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_I32I32toU8\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 int              isfmod,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out;\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_I32I32toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 int       isfmod,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src0 = read_imagei(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out;\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 int              isfmod,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    uint4 src0, src1;\n\
    float4 in0, in1, out;\n\
    READ_IMAGEUI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    uint4 dst  = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 int       isfmod,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    uint4 src0 = read_imageui(input, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
    float4 in0, in1, out;\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    uint4 dst  = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_U8I32toU8\n\
    (\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 int              isfmod,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail\n\
     )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    uint4 src0;\n\
    int4 src1;\n\
    float4 in0, in1, out;\n\
    READ_IMAGEUI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1);\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void mod_U8I32toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 int       isfmod,\n\
                 float     input0Scale,\n\
                 float     input0Tail,\n\
                 float     input1Scale,\n\
                 float     input1Tail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    uint4 src0 = read_imageui(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0, in1, out;\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1);\n\
    if (isfmod)\n\
    {\n\
        out = fmod(in0, in1) * outputScale + outputTail;\n\
    }\n\
    else\n\
    {\n\
        out = (in0 - in1 * floor(in0 / in1)) * outputScale + outputTail;\n\
    }\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
"; /* end of mod_cl*/

static const char moments_axis0_cl[] = "__kernel void moments_axis0_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(0, gidy, gidz, 0);\n\
    uint data;\n\
    float sum = 0, sqr = 0;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    {\n\
        for(coord0.x = 0; coord0.x < width;)\n\
        {\n\
            data = read_imageui(input, coord0).x;\n\
            coord0.x++;\n\
            tmpSum = tmpSum + data;\n\
            tmpSqr = tmpSqr + data * data;\n\
        }\n\
        sqr = convert_float(as_int(tmpSqr - 2 * input_zp * tmpSum + width * input_zp * input_zp)) * e2InScale;\n\
        sum = convert_float(as_int(tmpSum - width * input_zp)) * input_scale;\n\
    }\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
#define MOMENTS_AXIS0_F(src0_type_name) \\\n\
__kernel void moments_axis0_##src0_type_name##to##src0_type_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_t  output_mean, \\\n\
    __write_only image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
 \\\n\
    int4 coord0 = (int4)(0, gidy, gidz, 0); \\\n\
    float data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    for(coord0.x = 0; coord0.x < width;) \\\n\
    { \\\n\
        data = read_imagef(input, coord0).x; \\\n\
        coord0.x++; \\\n\
        sum += (data); \\\n\
        sqr += (data * data); \\\n\
    } \\\n\
 \\\n\
    float4 mean, vari; \\\n\
    mean.x = sum * dimRatio; \\\n\
    vari.x = sqr * dimRatio; \\\n\
    vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidy, gidz); \\\n\
    write_imagef(output_mean, coord_out, mean); \\\n\
    write_imagef(output_vari, coord_out, vari); \\\n\
}\n\
MOMENTS_AXIS0_F(F32)\n\
\n\
__kernel void moments_axis0_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(0, gidy, gidz, 0);\n\
    float data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.x = 0; coord0.x < width;)\n\
    {\n\
        data = convert_float(read_imagei(input, coord0).x - input_zp);\n\
        coord0.x++;\n\
\n\
        sum = sum + data;\n\
        sqr = sqr + data * data;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio * input_scale;\n\
    vari.x = sqr * dimRatio * input_scale * input_scale;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
__kernel void moments_axis0_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(0, gidy, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.x = 0; coord0.x < width;)\n\
    {\n\
        uint4 src0 = read_imageui(input, coord0);\n\
        src0 = src0 << 16;\n\
        _viv_asm(COPY, data, src0, 16);\n\
        coord0.x++;\n\
\n\
        sum = sum + data.x;\n\
        sqr = sqr + data.x * data.x;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}"; /* end of moments_axis0_cl*/

static const char moments_axis01_cl[] = "__kernel void moments_axis01_U8toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    uint4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        int tmpSum = 0, tmpSqr = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imageui(input, coord);\n\
            coord.y++;\n\
            tmpSum = tmpSum + data.x;\n\
            tmpSqr = tmpSqr + data.x * data.x;\n\
        }\n\
        sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum += (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
\n\
#define MOMENTS_AXIS01_F(src0_type_name) \\\n\
__kernel void moments_axis01_##src0_type_name##to##src0_type_name( \\\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int lidx = get_local_id(0); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    float4 data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            data = read_imagef(input, coord); \\\n\
            coord.y++; \\\n\
            sum += data.x; \\\n\
            sqr += data.x * data.x; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int2 coord_out = (int2)(gidz, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = 0; sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        write_imagef(output_mean, coord_out, mean); \\\n\
        write_imagef(output_vari, coord_out, vari); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_F(F32)\n\
\n\
__kernel void moments_axis01_I32toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        float tmpSum = 0, tmpSqr = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = convert_float4(read_imagei(input, coord));\n\
            coord.y++;\n\
\n\
            tmpSum = tmpSum + data.x;\n\
            tmpSqr = tmpSqr + data.x * data.x;\n\
        }\n\
        sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum += (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
\n\
__kernel void moments_axis01_BF16toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        float tmpSum = 0, tmpSqr = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            uint4 src0 = read_imageui(input, coord);\n\
            src0 = src0 << 16;\n\
            _viv_asm(COPY, data, src0, 16);\n\
            coord.y++;\n\
\n\
            tmpSum = tmpSum + data.x;\n\
            tmpSqr = tmpSqr + data.x * data.x;\n\
        }\n\
        sqr += tmpSqr;\n\
        sum += tmpSum;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(8, 8, 1))) void moments_axis12_U8toF32(\n\
    image2d_array_t   input, image2d_array_t  output_mean, image2d_array_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int lidy = get_local_id(1);\n\
    int gidz = get_global_id(2); // width\n\
\n\
    int4 coord = (int4)(gidz, lidx, lidy, 0);\n\
    uint4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local uint lcl_sumSqr[128];\n\
    __local uint lcl_sumSqr1[32];\n\
\n\
    uint2 tmpSumSqr = 0;\n\
    for(coord.z = lidy; coord.z < chn; coord.z += 8)\n\
    {\n\
        for(coord.y = lidx; coord.y < height;)\n\
        {\n\
            data = read_imageui(input, coord);\n\
            coord.y += 8;\n\
            tmpSumSqr = tmpSumSqr + (uint2)(data.x, data.x * data.x);\n\
        }\n\
        //sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        //sum += (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    int index = lidx + lidy * 8;\n\
    vstore2(tmpSumSqr, index, lcl_sumSqr);\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(index < 16)\n\
    {\n\
        uint4 val0 = vload4(index, lcl_sumSqr);\n\
        uint4 val1 = vload4(index, lcl_sumSqr + 64);\n\
        val0 += val1;\n\
        uint2 val2 = val0.xy + val0.zw;\n\
        vstore2(val2, index, lcl_sumSqr1);\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(index == 0)\n\
    {\n\
        uint4 val0 = 0;\n\
        for(int i = 0; i < 8; i++)\n\
        {\n\
            val0 += vload4(i, lcl_sumSqr1);\n\
        }\n\
\n\
        float2 tmpVal = convert_float2(val0.xy + val0.zw);\n\
        sum = (tmpVal.x - height * chn * input_zp) * input_scale;\n\
        sqr = (tmpVal.y - 2 * input_zp * tmpVal.x + height * chn * input_zp * input_zp) * e2InScale;\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        write_imagef(output_mean, coord.xwww, mean);\n\
        write_imagef(output_vari, coord.xwww, vari);\n\
    }\n\
}\n\
"; /* end of moments_axis01_cl*/

static const char moments_axis012_cl[] = "__kernel void moments_axis012_U8toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    uint4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.z = 0; coord.z < chn; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
        {\n\
            int tmpSum = 0, tmpSqr = 0;\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                data = read_imageui(input, coord);\n\
                coord.y++;\n\
                tmpSum = tmpSum + data.x;\n\
                tmpSqr = tmpSqr + data.x * data.x;\n\
            }\n\
            sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
            sum += (tmpSum - height * input_zp) * input_scale;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
\n\
#define MOMENTS_AXIS012_F(src0_type_name) \\\n\
__kernel void moments_axis012_##src0_type_name##to##src0_type_name( \\\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int lidx = get_local_id(0); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, 0, 0, 0); \\\n\
    float4 data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.z = 0; coord.z < chn; coord.z++) \\\n\
    { \\\n\
        for(coord.x = gidx; coord.x < width; coord.x += 16) \\\n\
        { \\\n\
            for(coord.y = 0; coord.y < height;) \\\n\
            { \\\n\
                data = read_imagef(input, coord); \\\n\
                coord.y++; \\\n\
                sum += data.x; \\\n\
                sqr += data.x * data.x; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = 0; sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        write_imagef(output_mean, coord_out, mean); \\\n\
        write_imagef(output_vari, coord_out, vari); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS012_F(F32)\n\
\n\
__kernel void moments_axis012_I32toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    int4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.z = 0; coord.z < chn; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
        {\n\
            int tmpSum = 0, tmpSqr = 0;\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                data = read_imagei(input, coord);\n\
                coord.y++;\n\
                tmpSum = tmpSum + data.x;\n\
                tmpSqr = tmpSqr + data.x * data.x;\n\
            }\n\
            sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
            sum += (tmpSum - height * input_zp) * input_scale;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
\n\
__kernel void moments_axis012_BF16toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.z = 0; coord.z < chn; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
        {\n\
            float tmpSum = 0, tmpSqr = 0;\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                uint4 src0 = read_imageui(input, coord);\n\
                src0 = src0 << 16;\n\
                _viv_asm(COPY, data, src0, 16);\n\
                coord.y++;\n\
                tmpSum = tmpSum + data.x;\n\
                tmpSqr = tmpSqr + data.x * data.x;\n\
            }\n\
            sqr += tmpSqr;\n\
            sum += tmpSum;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
"; /* end of moments_axis012_cl*/

static const char moments_axis1_cl[] = "__kernel void moments_axis1_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, 0, gidz, 0);\n\
    uint data;\n\
    float sum = 0, sqr = 0;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    {\n\
        for(coord0.y = 0; coord0.y < height;)\n\
        {\n\
            data = read_imageui(input, coord0).x;\n\
            coord0.y++;\n\
            tmpSum = tmpSum + data;\n\
            tmpSqr = tmpSqr + data * data;\n\
        }\n\
        sqr = convert_float(as_int(tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp)) * e2InScale;\n\
        sum = convert_float(as_int(tmpSum - height * input_zp)) * input_scale;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
#define MOMENTS_AXIS1_F(src0_type_name) \\\n\
__kernel void moments_axis1_##src0_type_name##to##src0_type_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_t  output_mean, \\\n\
    __write_only image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
 \\\n\
    int4 coord0 = (int4)(gidx, 0, gidz, 0); \\\n\
    float data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    for(coord0.y = 0; coord0.y < height;) \\\n\
    { \\\n\
        data = read_imagef(input, coord0).x; \\\n\
        coord0.y++; \\\n\
        sum += (data); \\\n\
        sqr += (data * data); \\\n\
    } \\\n\
 \\\n\
    float4 mean, vari; \\\n\
    mean.x = sum * dimRatio; \\\n\
    vari.x = sqr * dimRatio; \\\n\
    vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidz); \\\n\
    write_imagef(output_mean, coord_out, mean); \\\n\
    write_imagef(output_vari, coord_out, vari); \\\n\
}\n\
MOMENTS_AXIS1_F(F32)\n\
\n\
__kernel void moments_axis1_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, 0, gidz, 0);\n\
    float data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.y = 0; coord0.y < height;)\n\
    {\n\
        data = convert_float(read_imagei(input, coord0).x - input_zp);\n\
        coord0.y++;\n\
        sum = sum + data;\n\
        sqr = sqr + data * data;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio * input_scale;\n\
    vari.x = sqr * dimRatio * input_scale * input_scale;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
__kernel void moments_axis1_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.y = 0; coord0.y < height;)\n\
    {\n\
        uint4 src0 = read_imageui(input, coord0);\n\
        src0 = src0 << 16;\n\
        _viv_asm(COPY, data, src0, 16);\n\
        coord0.y++;\n\
        sum = sum + data.x;\n\
        sqr = sqr + data.x * data.x;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}"; /* end of moments_axis1_cl*/

static const char moments_axis2_cl[] = "__kernel void moments_axis2_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, gidy, 0, 0);\n\
    uint data;\n\
    float sum = 0, sqr = 0;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    {\n\
        for(coord0.z = 0; coord0.z < chn;)\n\
        {\n\
            data = read_imageui(input, coord0).x;\n\
            coord0.z++;\n\
            tmpSum = tmpSum + data;\n\
            tmpSqr = tmpSqr + data * data;\n\
        }\n\
        sqr = as_int(tmpSqr - 2 * input_zp * tmpSum + chn * input_zp * input_zp) * e2InScale;\n\
        sum = tmpSum * input_scale;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio - input_zp * input_scale;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
#define MOMENTS_AXIS2_F(src0_type_name) \\\n\
__kernel void moments_axis2_##src0_type_name##to##src0_type_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_t  output_mean, \\\n\
    __write_only image2d_t  output_vari, \\\n\
    int axis, \\\n\
    int axis_num, \\\n\
    int input_zp, \\\n\
    float input_scale, \\\n\
    int width, \\\n\
    int height, \\\n\
    int chn, \\\n\
    float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord0 = (int4)(gidx, gidy, 0, 0); \\\n\
    float data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    for(coord0.z = 0; coord0.z < chn;) \\\n\
    { \\\n\
        data = read_imagef(input, coord0).x; \\\n\
        coord0.z++; \\\n\
        sum += (data); \\\n\
        sqr += (data * data); \\\n\
    } \\\n\
 \\\n\
    float4 mean, vari; \\\n\
    mean.x = sum * dimRatio; \\\n\
    vari.x = sqr * dimRatio; \\\n\
    vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidy); \\\n\
    write_imagef(output_mean, coord_out, mean); \\\n\
    write_imagef(output_vari, coord_out, vari); \\\n\
}\n\
MOMENTS_AXIS2_F(F32)\n\
\n\
__kernel void moments_axis2_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, gidy, 0, 0);\n\
    float data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.z = 0; coord0.z < chn;)\n\
    {\n\
        data = convert_float(read_imagei(input, coord0).x - input_zp);\n\
        coord0.z++;\n\
\n\
\n\
        sum = sum + data;\n\
        sqr = sqr + data * data;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio * input_scale;\n\
    vari.x = sqr * dimRatio * input_scale * input_scale;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
__kernel void moments_axis2_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, gidy, 0, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.z = 0; coord0.z < chn;)\n\
    {\n\
        uint4 src0 = read_imageui(input, coord0);\n\
        src0 = src0 << 16;\n\
        _viv_asm(COPY, data, src0, 16);\n\
        coord0.z++;\n\
\n\
        sum = sum + data.x;\n\
        sqr = sqr + data.x * data.x;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
"; /* end of moments_axis2_cl*/

static const char nearest_grid_sample_cl[] = "__kernel void nearest_grid_sample_F32_F32toF32(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                           float  half_input0_w,\n\
                           float  half_input0_h,\n\
                           float  add_float_value_w,\n\
                           float  add_float_value_h,\n\
                           int    depth\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int2   coord_in1    =  (int2)(get_global_id(0) * 2, get_global_id(1));\n\
\n\
    float fx = read_imagef(input1, coord_in1).x;\n\
    coord_in1.x = coord_in1.x + 1;\n\
    float fy = read_imagef(input1, coord_in1).x;\n\
\n\
    fx = fx * half_input0_w + add_float_value_w;\n\
    fy = fy * half_input0_h + add_float_value_h;\n\
    int   x_index = convert_int(fx);\n\
    int   y_index = convert_int(fy);\n\
    int4   coord_in     = (int4)(x_index, y_index, 0, 0);\n\
\n\
    float4  dst;\n\
\n\
    while (coord_in.z < depth){\n\
        dst    = read_imagef(input0, coord_in);\n\
        write_imagef(output, coord_out, dst);\n\
        coord_in.z++;\n\
        coord_out.z++;\n\
    }\n\
}\n\
\n\
\n\
__kernel void nearest_grid_sample_U8_U8toU8(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                           float  half_input0_w,\n\
                           float  half_input0_h,\n\
                           float  add_float_value_w,\n\
                           float  add_float_value_h,\n\
                           int    depth,\n\
                           float  in0_scale,\n\
                           float  in0_tail,\n\
                           float  in1_scale,\n\
                           float  in1_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int2   coord_in1    =  (int2)(get_global_id(0) * 2, get_global_id(1));\n\
\n\
    float fx    = convert_float4(read_imageui(input1, coord_in1)).x * in1_scale + in1_tail;\n\
    coord_in1.x = coord_in1.x + 1;\n\
    float fy    = convert_float4(read_imageui(input1, coord_in1)).x * in1_scale + in1_tail;\n\
\n\
    fx = fx * half_input0_w + add_float_value_w;\n\
    fy = fy * half_input0_h + add_float_value_h;\n\
    int   x_index = convert_int(fx);\n\
    int   y_index = convert_int(fy);\n\
    int4   coord_in     = (int4)(x_index, y_index, 0, 0);\n\
\n\
    float4 val;\n\
    uint4  dst;\n\
\n\
    while (coord_in.z < depth){\n\
        val    = convert_float4(read_imageui(input0, coord_in)) * in0_scale + in0_tail;\n\
        dst      = convert_uint4_rte(val * out_scale + out_tail);\n\
        write_imageui(output, coord_out, dst);\n\
        coord_in.z++;\n\
        coord_out.z++;\n\
    }\n\
\n\
}\n\
"; /* end of nearest_grid_sample_cl*/

static const char one_hot_cl[] = "__kernel void one_hot_F32toF32\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     float           on_value,\n\
                     float           off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    float4 val = read_imagef(input, coord.xy);\n\
\n\
    do\n\
    {\n\
        float4 dst;\n\
        dst.x = convert_int(val.x) == coord.z ? on_value : off_value;\n\
\n\
        write_imagef(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_I32toI32\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     int             on_value,\n\
                     int             off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 src = read_imagei(input, coord.xy);\n\
\n\
    int  val = convert_int(convert_float(src.x) * inputScale - inputTail);\n\
    do\n\
    {\n\
        int4 dst;\n\
        dst.x = val == coord.z ? on_value : off_value;\n\
\n\
        write_imagei(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_I32toU8\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     uint            on_value,\n\
                     uint            off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 src = read_imagei(input, coord.xy);\n\
\n\
    int  val = convert_int(convert_float(src.x) * inputScale - inputTail);\n\
    do\n\
    {\n\
        uint4 dst;\n\
        dst.x = val == coord.z ? on_value : off_value;\n\
\n\
        write_imageui(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_I32toF32\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     float           on_value,\n\
                     float           off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 src = read_imagei(input, coord.xy);\n\
\n\
    int  val = convert_int(convert_float(src.x) * inputScale - inputTail);\n\
    do\n\
    {\n\
        float4 dst;\n\
        dst.x = val == coord.z ? on_value : off_value;\n\
\n\
        write_imagef(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_U8toU8\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     uint            on_value,\n\
                     uint            off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    uint4 src = read_imageui(input, coord.xy);\n\
\n\
    int  val = convert_int(convert_float(src.x) * inputScale - inputTail);\n\
\n\
    do\n\
    {\n\
        uint4 dst;\n\
        dst.x = val == coord.z ? on_value : off_value;\n\
\n\
        write_imageui(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_I32toBF16\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     uint            on_value,\n\
                     uint            off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 src = read_imagei(input, coord.xy);\n\
\n\
    int  val = convert_int(convert_float(src.x) * inputScale - inputTail);\n\
    do\n\
    {\n\
        uint4 dst;\n\
        dst.x = val == coord.z ? on_value : off_value;\n\
\n\
        write_imageui(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}"; /* end of one_hot_cl*/

static const char poolwithargmax_cl[] = "\n\
#define POOLWITHARGMAX_PROCESS(data_type, read_fun, write_fun0, write_fun1) \\\n\
    data_type src  = 0; \\\n\
    data_type max  = 0; \\\n\
    uint4  axis = 0; \\\n\
    src.x = read_fun(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_fun(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_fun(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_fun(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    write_fun0(output,  coord_out, max); \\\n\
    write_fun1(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_F32to_F32_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_PROCESS(float4, read_imagef, write_imagef, write_imageui)\n\
}\n\
\n\
__kernel void poolwithargmax_F32to_F32_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_PROCESS(float4, read_imagef, write_imagef, write_imageui)\n\
}\n\
\n\
__kernel void poolwithargmax_I32to_I32_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_PROCESS(int4, read_imagei, write_imagei, write_imageui)\n\
}\n\
\n\
__kernel void poolwithargmax_I32to_I32_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_PROCESS(int4, read_imagei, write_imagei, write_imageui)\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_U8_PROCESS() \\\n\
    uint4 src  = 0; \\\n\
    uint4 max  = 0; \\\n\
    uint4 axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x = read_imageui(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_imageui(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_imageui(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_imageui(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    result.x = convert_float4(max).x * scale_value + tail_value; \\\n\
    max = convert_uint4(result);\\\n\
    write_imageui(output,  coord_out, max); \\\n\
    write_imageui(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis,\n\
                           float   scale_value,\n\
                           float   tail_value)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_U8_PROCESS()\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis,\n\
                     float   scale_value,\n\
                     float   tail_value)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_U8_PROCESS()\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_U8_TO_F32_PROCESS() \\\n\
    uint4 src  = 0; \\\n\
    uint4 max  = 0; \\\n\
    uint4 axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x = read_imageui(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_imageui(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_imageui(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_imageui(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    result.x = convert_float4(max).x * scale_value + tail_value; \\\n\
    write_imagef(output,  coord_out, result); \\\n\
    write_imageui(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_U8to_F32_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis,\n\
                           float   scale_value,\n\
                           float   tail_value)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_U8_TO_F32_PROCESS()\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_F32_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis,\n\
                     float   scale_value,\n\
                     float   tail_value)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_U8_TO_F32_PROCESS()\n\
}\n\
\n\
#define POOLWITHARGMAX_F32_TO_U8_PROCESS() \\\n\
    float4 src  = 0; \\\n\
    float4 max  = 0; \\\n\
    uint4 axis = 0; \\\n\
    uint4 dst  = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x = read_imagef(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_imagef(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_imagef(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_imagef(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    result.x = max.x * scale_value + tail_value; \\\n\
    dst = convert_uint4(result);\\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    write_imageui(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_F32to_U8_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis,\n\
                           float   scale_value,\n\
                           float   tail_value)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_F32_TO_U8_PROCESS()\n\
}\n\
\n\
__kernel void poolwithargmax_F32to_U8_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis,\n\
                     float   scale_value,\n\
                     float   tail_value)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_F32_TO_U8_PROCESS()\n\
}\n\
"; /* end of poolwithargmax_cl*/

static const char pow_cl[] = "__kernel void pow_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              inputScale,\n\
                 float              inputTail,\n\
                 float              outputScale,\n\
                 float              outputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0, src1;\n\
    float4 dst;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4  s0 = sign(src0);\n\
    int4 t0 = convert_int4(src1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    dst.x = (src0.x == 0 && src1.x == 0) ? 1.0f : (src0.x != 0 ?\n\
         (s0.x * exp2(src1.x * log2(fabs(src0.x)))) : 0.0f);\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void pow_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        inputScale,\n\
                 float        inputTail,\n\
                 float        outputScale,\n\
                 float        outputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 dst = (float4)(0);\n\
\n\
    float4  s0 = sign(src0);\n\
    int4 t0 = convert_int4(src1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
\n\
    dst.x = (src0.x == 0 && src1.x == 0) ? 1.0f : (src0.x != 0 ?\n\
           (s0.x * exp2(src1.x * log2(fabs(src0.x)))) : 0.0f);\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void pow_U32F32toU32(\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              inputScale,\n\
                 float              inputTail,\n\
                 float              outputScale,\n\
                 float              outputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 src0, dst;\n\
    float4 src0_f, src1, dst_f;\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    src0_f = convert_float4(src0) * inputScale + inputTail;\n\
    float4  s0 = sign(src0_f);\n\
    int4 t0 = convert_int4(src1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    dst_f.x = (src0.x == 0 && src1.x == 0) ? 1.0f : (src0.x != 0 ?\n\
           (s0.x * exp2(src1.x * log2(fabs(src0.x)))) : 0.0f);\n\
    dst.x = convert_uint(dst_f.x * outputScale + outputTail);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void pow_U32F32toU32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        inputScale,\n\
                 float        inputTail,\n\
                 float        outputScale,\n\
                 float        outputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    uint4  src0 = read_imageui(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 src0_f = (float4)(0);\n\
    float4 dst_f  = (float4)(0);\n\
    uint4  dst    = (uint4)(0);\n\
\n\
    src0_f.x = convert_float(src0.x) * inputScale + inputTail;\n\
    float4  s0 = sign(src0_f);\n\
    int4 t0 = convert_int4(src1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
\n\
    dst_f.x = (src0.x == 0 && src1.x == 0) ? 1.0f : (src0.x != 0 ?\n\
         (s0.x * exp2(src1.x * log2(fabs(src0.x)))) : 0.0f);\n\
    dst.x = convert_uint(dst_f.x * outputScale + outputTail);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
"; /* end of pow_cl*/

static const char prelu_cl[] = "__kernel void prelu_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4 maxData = src0 >= 0 ? src0 : 0;\n\
    float4 minData = src0 < 0 ? src0 : 0;\n\
    float4 dst = maxData + minData * src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 maxData = src0 >= 0 ? src0 : 0;\n\
    float4 minData = src0 < 0 ? src0 : 0;\n\
    float4 dst = maxData + minData * src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 src0;\n\
    uint4 src1;\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    uint4 src0 = read_imageui(input0, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
\n\
__kernel void prelu_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    int4 dst = convert_int4(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    int4 src0 = read_imagei(input0, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    int4 dst = convert_int4(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
"; /* end of prelu_cl*/

static const char random_multinomial_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
\n\
uint4 _philox4x32bumpkey(uint4 key)\n\
{\n\
    uint4 mask = (uint4)((uint)0x9E3779B9, (uint)0xBB67AE85, 0, 0);\n\
    //key.x += ((uint)0x9E3779B9);\n\
    //key.y += ((uint)0xBB67AE85);\n\
    key += mask;\n\
    return key;\n\
}\n\
\n\
uint mullo32(uint a, uint b)\n\
{\n\
    return a * b;\n\
}\n\
\n\
uint mulhi32(uint a, uint b)\n\
{\n\
    return mul_hi(a, b);\n\
}\n\
\n\
uint4 _philox4x32round(uint4 ctr, uint4 key)\n\
{\n\
    uint PHILOX_M4x32_0 = ((uint)0xD2511F53);\n\
    uint PHILOX_M4x32_1 = ((uint)0xCD9E8D57);\n\
\n\
    uint lo0 = mullo32(PHILOX_M4x32_0, ctr.x);\n\
    uint hi0 = mulhi32(PHILOX_M4x32_0, ctr.x);\n\
    uint lo1 = mullo32(PHILOX_M4x32_1, ctr.z);\n\
    uint hi1 = mulhi32(PHILOX_M4x32_1, ctr.z);\n\
\n\
    uint4 out = (uint4)(hi1^ctr.y^key.x, lo1, hi0^ctr.w^key.y, lo0);\n\
    return out;\n\
}\n\
\n\
uint4 philox4x32_R_10(uint4 ctr, uint4 key)\n\
{\n\
    uint i;\n\
    ctr = _philox4x32round(ctr, key);\n\
    for (i = 1; i < 10; i++)\n\
    {\n\
        key = _philox4x32bumpkey(key);\n\
        ctr = _philox4x32round(ctr, key);\n\
    }\n\
    return ctr;\n\
}\n\
\n\
__kernel void random_seed(\n\
    __read_only  image2d_array_t  seeds,\n\
    __write_only image2d_array_t  output,\n\
                 int              iter,\n\
                 float            re_rand_max\n\
    )\n\
{\n\
    Tensor s_tensor = create_tensor_from_image2d_array(seeds, 4);\n\
    __global uint* seeds_ptr = (__global uint*)s_tensor.ptr;\n\
    seeds_ptr = seeds_ptr;\n\
    uint4 key = vload4(0, seeds_ptr);\n\
\n\
    uint4 ctr = (uint4)(0);\n\
    float4 result = 0;\n\
\n\
    Tensor o_tensor = create_tensor_from_image2d_array(output, 4);\n\
    __global float* output_ptr = (__global float*)o_tensor.ptr;\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        ctr = philox4x32_R_10(ctr, key);\n\
        result = convert_float4(ctr) * re_rand_max;\n\
        vstore4(result, i, output_ptr);\n\
    }\n\
}\n\
\n\
#define logE    (1.44269502f)\n\
float eltwise_unary_exp(float x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
// N times of 8\n\
// x dim = 1\n\
\n\
__kernel void random_multinomial_cdf_F32\n\
    (\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(1));\n\
    int class_max_iter = get_image_width(input);\n\
    float4 src0, data;\n\
    float4 dst = 0;\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
\n\
    for(coord.x = 1; coord.x < class_max_iter;)\n\
    {\n\
        src0 = read_imagef(input, coord);\n\
        coord.x ++;\n\
\n\
        maxVal = maxVal > src0 ? maxVal : src0;\n\
    }\n\
\n\
    for(coord.x = 0; coord.x < class_max_iter; )\n\
    {\n\
        float4 val;\n\
        src0 = read_imagef(input, coord);\n\
\n\
        data = src0 - maxVal;\n\
        val.x = eltwise_unary_exp(data.x);\n\
        val.x += dst.x;\n\
        dst.x = val.x;\n\
        write_imagef(output, coord.xy, val);\n\
        coord.x ++;\n\
    }\n\
}\n\
\n\
uint upper_bound(float* a, int n, float x)\n\
{\n\
    uint l = 0;\n\
    uint h = n;\n\
    while (l < h) {\n\
        int mid = (l + h) >> 1;\n\
        if (x >= a[mid]) {\n\
            l = mid + 1;\n\
        } else {\n\
            h = mid;\n\
        }\n\
    }\n\
    return l;\n\
}\n\
\n\
// one thread calculate 4\n\
__kernel void random_multinomial\n\
    (\n\
    __read_only image2d_array_t randoms,\n\
    __read_only image2d_array_t cdfs,\n\
   __write_only image2d_array_t output\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
    int class_size = get_image_width(cdfs);\n\
\n\
    int offset = gidy * class_size;\n\
    Tensor cdf_tensor = create_tensor_from_image2d_array(cdfs, 4);\n\
    __global float* cdf_ptr = (__global uint*)cdf_tensor.ptr;\n\
    __global float* cdfPtr = cdf_ptr + offset;\n\
\n\
    int width = get_image_width(randoms);\n\
    offset = coord.x + coord.y * width;\n\
    Tensor r_tensor = create_tensor_from_image2d_array(randoms, 4);\n\
    __global float* randoms_ptr = (__global float*)r_tensor.ptr;\n\
    randoms_ptr = randoms_ptr + offset;\n\
\n\
    width = get_image_width(output);\n\
    offset = coord.x + coord.y * width;\n\
    Tensor o_tensor = create_tensor_from_image2d_array(output, 4);\n\
    __global uint* output_ptr = (__global uint*)o_tensor.ptr;\n\
    output_ptr = output_ptr + offset;\n\
\n\
    float4 ran = vload4(0, randoms_ptr);\n\
    float total = cdfPtr[class_size - 1];\n\
    float4 target = ran * total;\n\
\n\
    uint4 out_class = (uint4)(0);\n\
    out_class.x = upper_bound(cdfPtr, class_size, target.x);\n\
    out_class.y = upper_bound(cdfPtr, class_size, target.y);\n\
    out_class.z = upper_bound(cdfPtr, class_size, target.z);\n\
    out_class.w = upper_bound(cdfPtr, class_size, target.w);\n\
\n\
    vstore4(out_class, 0, output_ptr);\n\
}\n\
\n\
"; /* end of random_multinomial_cl*/

static const char reduceall_internal_axis0_cl[] = "__kernel void reduceall_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 allVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.x ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    write_imagei(output, coord.yz, allVal);\n\
}\n\
\n\
__kernel void reduceall_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 allVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.x ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, allVal);\n\
}\n\
\n\
"; /* end of reduceall_internal_axis0_cl*/

static const char reduceall_internal_axis1_cl[] = "__kernel void reduceall_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 allVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.y ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    write_imagei(output, coord.xz, allVal);\n\
}\n\
\n\
__kernel void reduceall_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 allVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.y ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    coord.y = 0;\n\
    write_imagei(output, coord, allVal);\n\
}\n\
\n\
"; /* end of reduceall_internal_axis1_cl*/

static const char reduceall_internal_axis2_cl[] = "__kernel void reduceall_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 allVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.z ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    write_imagei(output, coord.xy, allVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reduceall_internal_axis2_cl*/

static const char reduceany_internal_axis0_cl[] = "__kernel void reduceany_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 anyVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.x ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    write_imagei(output, coord.yz, anyVal);\n\
}\n\
\n\
__kernel void reduceany_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 anyVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.x ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, anyVal);\n\
}\n\
\n\
"; /* end of reduceany_internal_axis0_cl*/

static const char reduceany_internal_axis1_cl[] = "__kernel void reduceany_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 anyVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.y ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    write_imagei(output, coord.xz, anyVal);\n\
}\n\
\n\
__kernel void reduceany_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 anyVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.y ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    coord.y = 0;\n\
    write_imagei(output, coord, anyVal);\n\
}\n\
\n\
"; /* end of reduceany_internal_axis1_cl*/

static const char reduceany_internal_axis2_cl[] = "__kernel void reduceany_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 anyVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.z ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    write_imagei(output, coord.xy, anyVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reduceany_internal_axis2_cl*/

static const char reducemax_internal_axis0_cl[] = "__kernel void reducemax_axis0_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord    =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize  = get_image_width(input);\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagef(output, coord.yz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis0_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagef(output, coord.yx, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis0_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.yz, dst);\n\
}\n\
\n\
__kernel void reducemax_axis0_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    coord.x = 0;\n\
    write_imageui(output, coord.yx, dst);\n\
}\n\
\n\
__kernel void reducemax_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 maxVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 maxVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, maxVal);\n\
}\n\
\n\
"; /* end of reducemax_internal_axis0_cl*/

static const char reducemax_internal_axis1_cl[] = "__kernel void reducemax_axis1_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis1_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagef(output, coord, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis1_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xz, dst);\n\
}\n\
\n\
__kernel void reducemax_axis1_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    coord.y = 0;\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void reducemax_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 maxVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 maxVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, maxVal);\n\
}\n\
\n\
"; /* end of reducemax_internal_axis1_cl*/

static const char reducemax_internal_axis2_cl[] = "__kernel void reducemax_axis2_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, maxVal);\n\
}\n\
\n\
\n\
__kernel void reducemax_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.z ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xy, dst);\n\
}\n\
\n\
\n\
__kernel void reducemax_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 maxVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, maxVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reducemax_internal_axis2_cl*/

static const char reducemin_internal_axis0_cl[] = "__kernel void reducemin_axis0_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord    =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize  = get_image_width(input);\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagef(output, coord.yz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis0_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagef(output, coord.yx, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis0_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.yz, dst);\n\
}\n\
\n\
__kernel void reducemin_axis0_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    coord.x = 0;\n\
    write_imageui(output, coord.yx, dst);\n\
}\n\
\n\
__kernel void reducemin_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 minVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 minVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minVal);\n\
}\n\
\n\
"; /* end of reducemin_internal_axis0_cl*/

static const char reducemin_internal_axis1_cl[] = "__kernel void reducemin_axis1_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis1_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagef(output, coord, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis1_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xz, dst);\n\
}\n\
\n\
__kernel void reducemin_axis1_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    coord.y = 0;\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void reducemin_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minVal);\n\
}\n\
\n\
"; /* end of reducemin_internal_axis1_cl*/

static const char reducemin_internal_axis2_cl[] = "__kernel void reducemin_axis2_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, minVal);\n\
}\n\
\n\
\n\
__kernel void reducemin_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xy, dst);\n\
}\n\
\n\
\n\
__kernel void reducemin_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reducemin_internal_axis2_cl*/

static const char reduceprod_internal_axis0_cl[] = "__kernel void reduceprod_axis0_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord    =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize  = get_image_width(input);\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagef(output, coord.yz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis0_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagef(output, coord.yx, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis0_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    write_imageui(output, coord.yz, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis0_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    coord.x = 0;\n\
    write_imageui(output, coord.yx, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 prodVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 prodVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, prodVal);\n\
}\n\
\n\
"; /* end of reduceprod_internal_axis0_cl*/

static const char reduceprod_internal_axis1_cl[] = "__kernel void reduceprod_axis1_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis1_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagef(output, coord, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis1_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    write_imageui(output, coord.xz, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis1_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    coord.y = 0;\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 prodVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 prodVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, prodVal);\n\
}\n\
\n\
"; /* end of reduceprod_internal_axis1_cl*/

static const char reduceprod_internal_axis2_cl[] = "__kernel void reduceprod_axis2_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, prodVal);\n\
}\n\
\n\
\n\
__kernel void reduceprod_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.z ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    write_imageui(output, coord.xy, dst);\n\
}\n\
\n\
\n\
__kernel void reduceprod_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 prodVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, prodVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reduceprod_internal_axis2_cl*/

static const char relational_ops_cl[] = "\n\
#define COMPARISONS_F32(func_name, comp_op) \\\n\
__kernel void func_name##_F32F32toBOOL8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input0Scale, \\\n\
                 float           input0Tail, \\\n\
                 float           input1Scale, \\\n\
                 float           input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src0; \\\n\
    float4 src1; \\\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord); \\\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_F32(less, <)\n\
COMPARISONS_F32(great, >)\n\
COMPARISONS_F32(less_equal, <=)\n\
COMPARISONS_F32(great_equal, >=)\n\
COMPARISONS_F32(equal, ==)\n\
COMPARISONS_F32(not_equal, !=)\n\
\n\
#define COMPARISONS_F32_2D(func_name, comp_op) \\\n\
__kernel void func_name##_F32F32toBOOL8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input0Scale, \\\n\
                 float     input0Tail, \\\n\
                 float     input1Scale, \\\n\
                 float     input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src0 = read_imagef(input0, coord); \\\n\
    float4 src1 = read_imagef(input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_F32_2D(less, <)\n\
COMPARISONS_F32_2D(great, >)\n\
COMPARISONS_F32_2D(less_equal, <=)\n\
COMPARISONS_F32_2D(great_equal, >=)\n\
COMPARISONS_F32_2D(equal, ==)\n\
COMPARISONS_F32_2D(not_equal, !=)\n\
\n\
#define COMPARISONS_U32(func_name, comp_op) \\\n\
__kernel void func_name##_U32U32toBOOL8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input0Scale, \\\n\
                 float           input0Tail, \\\n\
                 float           input1Scale, \\\n\
                 float           input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 data0; \\\n\
    uint4 data1; \\\n\
    READ_IMAGEUI_2DARRAY(data0, input0, coord); \\\n\
    READ_IMAGEUI_2DARRAY(data1, input1, coord); \\\n\
 \\\n\
    float4 src0 = convert_float4(data0) * input0Scale - input0Tail; \\\n\
    float4 src1 = convert_float4(data1) * input1Scale - input1Tail; \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_U32(less, <)\n\
COMPARISONS_U32(great, >)\n\
COMPARISONS_U32(less_equal, <=)\n\
COMPARISONS_U32(great_equal, >=)\n\
COMPARISONS_U32(equal, ==)\n\
COMPARISONS_U32(not_equal, !=)\n\
\n\
#define COMPARISONS_U32_2D(func_name, comp_op) \\\n\
__kernel void func_name##_U32U32toBOOL8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input0Scale, \\\n\
                 float     input0Tail, \\\n\
                 float     input1Scale, \\\n\
                 float     input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 data0 = read_imageui(input0, coord); \\\n\
    uint4 data1 = read_imageui(input1, coord); \\\n\
 \\\n\
    float4 src0 = convert_float4(data0) * input0Scale - input0Tail; \\\n\
    float4 src1 = convert_float4(data1) * input1Scale - input1Tail; \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_U32_2D(less, <)\n\
COMPARISONS_U32_2D(great, >)\n\
COMPARISONS_U32_2D(less_equal, <=)\n\
COMPARISONS_U32_2D(great_equal, >=)\n\
COMPARISONS_U32_2D(equal, ==)\n\
COMPARISONS_U32_2D(not_equal, !=)\n\
\n\
#define COMPARISONS_I32(func_name, comp_op) \\\n\
__kernel void func_name##_I32I32toBOOL8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input0Scale, \\\n\
                 float           input0Tail, \\\n\
                 float           input1Scale, \\\n\
                 float           input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    int4 src0; \\\n\
    int4 src1; \\\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord); \\\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_I32(less, <)\n\
COMPARISONS_I32(great, >)\n\
COMPARISONS_I32(less_equal, <=)\n\
COMPARISONS_I32(great_equal, >=)\n\
COMPARISONS_I32(equal, ==)\n\
COMPARISONS_I32(not_equal, !=)\n\
\n\
#define COMPARISONS_I32_2D(func_name, comp_op) \\\n\
__kernel void func_name##_I32I32toBOOL8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input0Scale, \\\n\
                 float     input0Tail, \\\n\
                 float     input1Scale, \\\n\
                 float     input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    int4 src0 = read_imagei(input0, coord); \\\n\
    int4 src1 = read_imagei(input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_I32_2D(less, <)\n\
COMPARISONS_I32_2D(great, >)\n\
COMPARISONS_I32_2D(less_equal, <=)\n\
COMPARISONS_I32_2D(great_equal, >=)\n\
COMPARISONS_I32_2D(equal, ==)\n\
COMPARISONS_I32_2D(not_equal, !=)\n\
\n\
"; /* end of relational_ops_cl*/

static const char relu_keras_cl[] = "\n\
__kernel void relu_keras_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_F32toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_F32toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset,\n\
                           float  inputScale,\n\
                           float  inputTail,\n\
                           float  outputScale,\n\
                           float  outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_F32toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset,\n\
                     float  inputScale,\n\
                     float  inputTail,\n\
                     float  outputScale,\n\
                     float  outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset,\n\
                           float  inputScale,\n\
                           float  inputTail,\n\
                           float  outputScale,\n\
                           float  outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset,\n\
                     float  inputScale,\n\
                     float  inputTail,\n\
                     float  outputScale,\n\
                     float  outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset,\n\
                           float  inputScale,\n\
                           float  inputTail,\n\
                           float  outputScale,\n\
                           float  outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset,\n\
                     float  inputScale,\n\
                     float  inputTail,\n\
                     float  outputScale,\n\
                     float  outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}"; /* end of relu_keras_cl*/

static const char repeat_cl[] = "__kernel void repeat_I32_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.y = 0; coord.y < height;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.yw);\n\
        coord.y++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.y++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_I32_axis1(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xw);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_I32_axis2(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.z = 0; coord.z < channel;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.zw);\n\
        coord.z++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.z++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_I32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int2 coord = (int2)(0, 0);\n\
    int2 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xy);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.y = 0; coord.y < height;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.yw);\n\
        coord.y++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.y++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_axis1(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xw);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_axis2(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.z = 0; coord.z < channel;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.zw);\n\
        coord.z++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.z++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int2 coord = (int2)(0, 0);\n\
    int2 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xy);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
"; /* end of repeat_cl*/

static const char resize_1d_bilinear_cl[] = "__kernel void resize_1d_bilinear_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    int4   coord_in     = (int4)(left_x_idx, coord_out.y, coord_out.z, 0);\n\
    float4 top_l, top_r, top, bottom, dst;\n\
\n\
    top_l    = read_imagef(input, coord_in);\n\
    coord_in.x++;\n\
    top_r    = read_imagef(input, coord_in);\n\
\n\
    top_r    = top_r - top_l;\n\
    dst      = top_l + x_lerp * top_r;\n\
\n\
    write_imagef(output, coord_out, dst);\n\
\n\
}\n\
\n\
\n\
__kernel void resize_1d_bilinear_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value,\n\
                           float  in_scale,\n\
                           float  in_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    int4   coord_in     = (int4)(left_x_idx, coord_out.y, coord_out.z, 0);\n\
    float4 top_l, top_r, top;\n\
    uint4  dst;\n\
\n\
    top_l    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    top_r    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
\n\
    top_r    = top_r - top_l;\n\
    top      = top_l + x_lerp * top_r;\n\
    dst      = convert_uint4(top * out_scale + out_tail);\n\
\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_1d_bilinear_cl*/

static const char resize_1d_nearest_cl[] = "\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    float  in_x       = (convert_float(coord_out.x) + half_pixel_value) * scale_x + round_value; \\\n\
    int    in_x_idx   = convert_int(in_x); \\\n\
\n\
__kernel void resize_1d_nearest_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value,\n\
                           float  round_value)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, coord_out.y, coord_out.z, 0);\n\
    float4 dst;\n\
    dst    = read_imagef(input, coord_in);\n\
    write_imagef(output, coord_out, dst);\n\
}\n\
\n\
\n\
__kernel void resize_1d_nearest_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, coord_out.y, coord_out.z, 0);\n\
    uint4 dst;\n\
    dst    = convert_uint4(convert_float4(read_imageui(input, coord_in)) * output_scale + output_tail);\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_1d_nearest_cl*/

static const char resize_3d_bilinear_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
\n\
#define RESIZE_3D(in_name, out_name, read_image_type, dst_type, convert_type, write_image_type) \\\n\
__kernel void resize_3d_bilinear_##in_name##to##out_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  scale_x, \\\n\
                           float  scale_y, \\\n\
                           float  scale_z, \\\n\
                           float  half_pixel_value, \\\n\
                           uint   in_width, \\\n\
                           uint   in_height, \\\n\
                           uint   in_depth, \\\n\
                           float  in_scale, \\\n\
                           float  in_tail, \\\n\
                           float  out_scale, \\\n\
                           float  out_tail \\\n\
                           ) \\\n\
{ \\\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value; \\\n\
    float  left_x_f     = fmax(floor(in_x), 0); \\\n\
    float  x_lerp       = in_x - left_x_f; \\\n\
    int    left_x_idx   = convert_int(left_x_f); \\\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value; \\\n\
    float  top_y_f      = fmax(floor(in_y), 0); \\\n\
    float  y_lerp       = in_y - top_y_f; \\\n\
    int    top_y_idx    = convert_int(top_y_f); \\\n\
    float  in_z         = (convert_float(coord_out.z) + half_pixel_value) * scale_z - half_pixel_value; \\\n\
    float  front_z_f    = fmax(floor(in_z), 0); \\\n\
    float  z_lerp       = in_z - front_z_f; \\\n\
    int    front_z_idx  = convert_int(front_z_f); \\\n\
    int4   coord_in     = (int4)(left_x_idx, top_y_idx, front_z_idx, 0); \\\n\
    float4 data_000, data_100, data_010, data_110, data_001, data_011, data_101, data_111; \\\n\
    dst_type  dst; \\\n\
 \\\n\
    int dx, dy, dz; \\\n\
    dx = in_x < 0 ? 0 : (left_x_f < in_width - 1 ? 1 : 0); \\\n\
    dy = in_y < 0 ? 0 : (top_y_f < in_height - 1 ? 1 : 0); \\\n\
    dz = in_z < 0 ? 0 : (front_z_idx < in_depth - 1 ? 1 : 0); \\\n\
 \\\n\
    data_000 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
    coord_in.y = coord_in.y + dy; \\\n\
    data_010 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
    coord_in.x = coord_in.x + dx; \\\n\
    data_110 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
    coord_in.y = coord_in.y - dy; \\\n\
    data_100 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
    coord_in.z = coord_in.z + dz; \\\n\
    data_101 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
    coord_in.y = coord_in.y + dy; \\\n\
    data_111 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
    coord_in.x = coord_in.x - dx; \\\n\
    data_011 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
    coord_in.y = coord_in.y - dy; \\\n\
    data_001 = convert_float4(read_image_type(input, coord_in)) * in_scale + in_tail; \\\n\
 \\\n\
    data_000 = data_000 + (data_100 - data_000) * x_lerp; \\\n\
    data_010 = data_010 + (data_110 - data_010) * x_lerp; \\\n\
    data_000 = data_000 + (data_010 - data_000) * y_lerp; \\\n\
 \\\n\
    data_001 = data_001 + (data_101 - data_001) * x_lerp; \\\n\
    data_011 = data_011 + (data_111 - data_011) * x_lerp; \\\n\
    data_001 = data_001 + (data_011 - data_001) * y_lerp; \\\n\
    data_000 = data_000 + (data_001 - data_000) * z_lerp; \\\n\
 \\\n\
    dst      = convert_type(data_000 * out_scale + out_tail); \\\n\
 \\\n\
    write_image_type(output, coord_out, dst); \\\n\
}\n\
RESIZE_3D(F32, F32, read_imagef,  float4, convert_float4, write_imagef)\n\
RESIZE_3D(F32, U8,  read_imagef,  uint4,  convert_uint4,  write_imageui)\n\
RESIZE_3D(U8,  F32, read_imageui, float4, convert_float4, write_imagef)\n\
RESIZE_3D(U8,  U8,  read_imageui, uint4,  convert_uint4,  write_imageui)\n\
RESIZE_3D(I8,  I8,  read_imagei,  int4,   convert_int4,   write_imagei)\n\
\n\
__kernel void resize_3d_bilinear_BF16toBF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  scale_z,\n\
                           float  half_pixel_value,\n\
                           uint   in_width,\n\
                           uint   in_height,\n\
                           uint   in_depth,\n\
                           float  in_scale,\n\
                           float  in_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = fmax(floor(in_x), 0);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value;\n\
    float  top_y_f      = fmax(floor(in_y), 0);\n\
    float  y_lerp       = in_y - top_y_f;\n\
    int    top_y_idx    = convert_int(top_y_f);\n\
    float  in_z         = (convert_float(coord_out.z) + half_pixel_value) * scale_z - half_pixel_value;\n\
    float  front_z_f    = fmax(floor(in_z), 0);\n\
    float  z_lerp       = in_z - front_z_f;\n\
    int    front_z_idx  = convert_int(front_z_f);\n\
    int4   coord_in     = (int4)(left_x_idx, top_y_idx, front_z_idx, 0);\n\
    uint4 data_000, data_100, data_010, data_110, data_001, data_011, data_101, data_111;\n\
    float4 data_000_f, data_100_f, data_010_f, data_110_f, data_001_f, data_011_f, data_101_f, data_111_f;\n\
    uint4  dst;\n\
\n\
    int dx, dy, dz;\n\
    dx = in_x < 0 ? 0 : (left_x_f < in_width - 1 ? 1 : 0);\n\
    dy = in_y < 0 ? 0 : (top_y_f < in_height - 1 ? 1 : 0);\n\
    dz = in_z < 0 ? 0 : (front_z_idx < in_depth - 1 ? 1 : 0);\n\
\n\
    data_000 = read_imageui(input, coord_in);\n\
    data_000 = data_000 << 16;\n\
    coord_in.y = coord_in.y + dy;\n\
    data_010 = read_imageui(input, coord_in);\n\
    data_010 = data_010 << 16;\n\
    coord_in.x = coord_in.x + dx;\n\
    data_110 = read_imageui(input, coord_in);\n\
    data_110 = data_110 << 16;\n\
    coord_in.y = coord_in.y - dy;\n\
    data_100 = read_imageui(input, coord_in);\n\
    data_100 = data_100 << 16;\n\
    coord_in.z = coord_in.z + dz;\n\
    data_101 = read_imageui(input, coord_in);\n\
    data_101 = data_101 << 16;\n\
    coord_in.y = coord_in.y + dy;\n\
    data_111 = read_imageui(input, coord_in);\n\
    data_111 = data_111 << 16;\n\
    coord_in.x = coord_in.x - dx;\n\
    data_011 = read_imageui(input, coord_in);\n\
    data_011 = data_011 << 16;\n\
    coord_in.y = coord_in.y - dy;\n\
    data_001 = read_imageui(input, coord_in);\n\
    data_001 = data_001 << 16;\n\
\n\
    _viv_asm(COPY, data_000_f, data_000, 16);\n\
    _viv_asm(COPY, data_010_f, data_010, 16);\n\
    _viv_asm(COPY, data_110_f, data_110, 16);\n\
    _viv_asm(COPY, data_100_f, data_100, 16);\n\
    _viv_asm(COPY, data_101_f, data_101, 16);\n\
    _viv_asm(COPY, data_111_f, data_111, 16);\n\
    _viv_asm(COPY, data_011_f, data_011, 16);\n\
    _viv_asm(COPY, data_001_f, data_001, 16);\n\
\n\
    data_000_f = data_000_f + (data_100_f - data_000_f) * x_lerp;\n\
    data_010_f = data_010_f + (data_110_f - data_010_f) * x_lerp;\n\
    data_000_f = data_000_f + (data_010_f - data_000_f) * y_lerp;\n\
\n\
    data_001_f = data_001_f + (data_101_f - data_001_f) * x_lerp;\n\
    data_011_f = data_011_f + (data_111_f - data_011_f) * x_lerp;\n\
    data_001_f = data_001_f + (data_011_f - data_001_f) * y_lerp;\n\
    data_000_f = data_000_f + (data_001_f - data_000_f) * z_lerp;\n\
\n\
    _viv_asm(COPY, dst, data_000_f, 16);\n\
    dst = dst >> 16;\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_3d_bilinear_cl*/

static const char resize_3d_nearest_cl[] = "\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    float  in_x       = (convert_float(coord_out.x) + half_pixel_value) * scale_x + round_value; \\\n\
    int    in_x_idx   = convert_int(in_x); \\\n\
    float  in_y       = (convert_float(coord_out.y) + half_pixel_value) * scale_y + round_value; \\\n\
    int    in_y_idx   = convert_int(in_y); \\\n\
    float  in_z       = (convert_float(coord_out.z) + half_pixel_value) * scale_z + round_value; \\\n\
    int    in_z_idx   = convert_int(in_z); \\\n\
\n\
__kernel void resize_3d_nearest_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  scale_z,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, in_z_idx, 0);\n\
    float4 dst;\n\
    dst    = read_imagef(input, coord_in);\n\
    write_imagef(output, coord_out, dst);\n\
}\n\
\n\
\n\
__kernel void resize_3d_nearest_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  scale_z,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, in_z_idx, 0);\n\
    uint4 dst;\n\
    dst    = convert_uint4(convert_float4(read_imageui(input, coord_in)) * output_scale + output_tail);\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
\n\
__kernel void resize_3d_nearest_U8toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  scale_z,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, in_z_idx, 0);\n\
    float4 dst;\n\
    dst    = convert_float4(read_imageui(input, coord_in)) * output_scale + output_tail;\n\
    write_imagef(output, coord_out, dst);\n\
}\n\
\n\
__kernel void resize_3d_nearest_F32toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  scale_z,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, in_z_idx, 0);\n\
    uint4 dst;\n\
    dst    = convert_uint4(read_imagef(input, coord_in) * output_scale + output_tail);\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
\n\
__kernel void resize_3d_nearest_I8toI8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  scale_z,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, in_z_idx, 0);\n\
    int4 dst;\n\
    dst    = convert_int4(convert_float4(read_imagei(input, coord_in)) * output_scale);\n\
    write_imagei(output, coord_out, dst);\n\
}\n\
\n\
__kernel void resize_3d_nearest_BF16toBF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  scale_z,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, in_z_idx, 0);\n\
    uint4 dst;\n\
    dst = read_imageui(input, coord_in);\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
\n\
"; /* end of resize_3d_nearest_cl*/

static const char resize_bilinear_cl[] = "__kernel void resize_bilinear_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float  y_lerp       = in_y - top_y_f;\n\
    int    top_y_idx    = convert_int(top_y_f);\n\
    int4   coord_in     = (int4)(left_x_idx, top_y_idx, coord_out.z, 0);\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom, dst;\n\
\n\
    top_l    = read_imagef(input, coord_in);\n\
    coord_in.y++;\n\
    bottom_l = read_imagef(input, coord_in);\n\
    coord_in.x++;\n\
    bottom_r = read_imagef(input, coord_in);\n\
    coord_in.y--;\n\
    top_r    = read_imagef(input, coord_in);\n\
\n\
    top_r    = top_r - top_l;\n\
    top      = top_l + x_lerp * top_r;\n\
    bottom_r = bottom_r - bottom_l;\n\
    bottom   = bottom_l + x_lerp * bottom_r;\n\
    bottom   = bottom - top;\n\
    dst      = top + y_lerp * bottom;\n\
\n\
    write_imagef(output, coord_out, dst);\n\
\n\
}\n\
\n\
\n\
__kernel void resize_bilinear_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value,\n\
                           float  in_scale,\n\
                           float  in_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float  y_lerp       = in_y - top_y_f;\n\
    int    top_y_idx    = convert_int(top_y_f);\n\
    int4   coord_in     = (int4)(left_x_idx, top_y_idx, coord_out.z, 0);\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom;\n\
    uint4  dst;\n\
\n\
    top_l    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.y++;\n\
    bottom_l = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    bottom_r = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.y--;\n\
    top_r    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
\n\
    top_r    = top_r - top_l;\n\
    top      = top_l + x_lerp * top_r;\n\
    bottom_r = bottom_r - bottom_l;\n\
    bottom   = bottom_l + x_lerp * bottom_r;\n\
    bottom   = bottom - top;\n\
    top      = top + y_lerp * bottom;\n\
\n\
    dst      = convert_uint4(top * out_scale + out_tail);\n\
\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_bilinear_cl*/

static const char resize_cubic_cl[] = "__kernel void resize_cubic_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  cubic_coeffs_y[4] = {0,0,0,0};\n\
    float  cubic_coeffs_x[4] = {0,0,0,0};\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float4 delta_x      = (float4)(0, in_x - left_x_f,0,0);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float4 delta_y      = (float4)(0, in_y - top_y_f,0,0);\n\
    int    x_idx        = convert_int(left_x_f - 1);\n\
    int    y_idx        = convert_int(top_y_f - 1);\n\
    int4   coord_in     = (int4)(x_idx, y_idx, coord_out.z, 0);\n\
    float  data00, data01, data02, data03, data10, data11, data12, data13,\n\
           data20, data21, data22, data23, data30, data31, data32, data33;\n\
\n\
    delta_x.x = 1 + delta_x.y;\n\
    delta_x.z = 1 - delta_x.y;\n\
    delta_x.w = 2 - delta_x.y;\n\
    cubic_coeffs_x[0] = -0.5 * ((((delta_x.x - 5) * delta_x.x + 8) * delta_x.x) - 4);\n\
    cubic_coeffs_x[1] = (1.5 * delta_x.y - 2.5) * delta_x.y * delta_x.y + 1;\n\
    cubic_coeffs_x[2] = (1.5 * delta_x.z - 2.5) * delta_x.z * delta_x.z + 1;\n\
    cubic_coeffs_x[3] = -0.5 * ((((delta_x.w - 5) * delta_x.w + 8) * delta_x.w) - 4);\n\
    delta_y.x = 1 + delta_y.y;\n\
    delta_y.z = 1 - delta_y.y;\n\
    delta_y.w = 2 - delta_y.y;\n\
    cubic_coeffs_y[0] = -0.5 * ((((delta_y.x - 5) * delta_y.x + 8) * delta_y.x) - 4);\n\
    cubic_coeffs_y[1] = (1.5 * delta_y.y - 2.5) * delta_y.y * delta_y.y + 1;\n\
    cubic_coeffs_y[2] = (1.5 * delta_y.z - 2.5) * delta_y.z * delta_y.z + 1;\n\
    cubic_coeffs_y[3] = -0.5 * ((((delta_y.w - 5) * delta_y.w + 8) * delta_y.w) - 4);\n\
    float4 dst = (float4)(0,0,0,0);\n\
\n\
    data00   = read_imagef(input, coord_in).x;\n\
    coord_in.x++;\n\
    data10   = read_imagef(input, coord_in).x;\n\
    coord_in.x++;\n\
    data20   = read_imagef(input, coord_in).x;\n\
    coord_in.x++;\n\
    data30   = read_imagef(input, coord_in).x;\n\
\n\
    coord_in.y++;\n\
    data31   = read_imagef(input, coord_in).x;\n\
    coord_in.x--;\n\
    data21   = read_imagef(input, coord_in).x;\n\
    coord_in.x--;\n\
    data11   = read_imagef(input, coord_in).x;\n\
    coord_in.x--;\n\
    data01   = read_imagef(input, coord_in).x;\n\
\n\
    coord_in.y++;\n\
    data02   = read_imagef(input, coord_in).x;\n\
    coord_in.x++;\n\
    data12   = read_imagef(input, coord_in).x;\n\
    coord_in.x++;\n\
    data22   = read_imagef(input, coord_in).x;\n\
    coord_in.x++;\n\
    data32   = read_imagef(input, coord_in).x;\n\
\n\
    coord_in.y++;\n\
    data33   = read_imagef(input, coord_in).x;\n\
    coord_in.x--;\n\
    data23   = read_imagef(input, coord_in).x;\n\
    coord_in.x--;\n\
    data13   = read_imagef(input, coord_in).x;\n\
    coord_in.x--;\n\
    data03   = read_imagef(input, coord_in).x;\n\
\n\
    dst.x = data00 * cubic_coeffs_x[0] * cubic_coeffs_y[0]\n\
          + data01 * cubic_coeffs_x[0] * cubic_coeffs_y[1]\n\
          + data02 * cubic_coeffs_x[0] * cubic_coeffs_y[2]\n\
          + data03 * cubic_coeffs_x[0] * cubic_coeffs_y[3]\n\
          + data10 * cubic_coeffs_x[1] * cubic_coeffs_y[0]\n\
          + data11 * cubic_coeffs_x[1] * cubic_coeffs_y[1]\n\
          + data12 * cubic_coeffs_x[1] * cubic_coeffs_y[2]\n\
          + data13 * cubic_coeffs_x[1] * cubic_coeffs_y[3]\n\
          + data20 * cubic_coeffs_x[2] * cubic_coeffs_y[0]\n\
          + data21 * cubic_coeffs_x[2] * cubic_coeffs_y[1]\n\
          + data22 * cubic_coeffs_x[2] * cubic_coeffs_y[2]\n\
          + data23 * cubic_coeffs_x[2] * cubic_coeffs_y[3]\n\
          + data30 * cubic_coeffs_x[3] * cubic_coeffs_y[0]\n\
          + data31 * cubic_coeffs_x[3] * cubic_coeffs_y[1]\n\
          + data32 * cubic_coeffs_x[3] * cubic_coeffs_y[2]\n\
          + data33 * cubic_coeffs_x[3] * cubic_coeffs_y[3];\n\
\n\
    write_imagef(output, coord_out, dst);\n\
\n\
}\n\
\n\
\n\
__kernel void resize_cubic_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value,\n\
                           float  in_scale,\n\
                           float  in_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  cubic_coeffs_y[4] = {0,0,0,0};\n\
    float  cubic_coeffs_x[4] = {0,0,0,0};\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float4 delta_x      = (float4)(0, in_x - left_x_f,0,0);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float4 delta_y      = (float4)(0, in_y - top_y_f,0,0);\n\
    int    x_idx        = convert_int(left_x_f - 1);\n\
    int    y_idx        = convert_int(top_y_f - 1);\n\
    int4   coord_in     = (int4)(x_idx, y_idx, coord_out.z, 0);\n\
    float  data00, data01, data02, data03, data10, data11, data12, data13,\n\
           data20, data21, data22, data23, data30, data31, data32, data33;\n\
\n\
    delta_x.x = 1 + delta_x.y;\n\
    delta_x.z = 1 - delta_x.y;\n\
    delta_x.w = 2 - delta_x.y;\n\
    cubic_coeffs_x[0] = -0.5 * ((((delta_x.x - 5) * delta_x.x + 8) * delta_x.x) - 4);\n\
    cubic_coeffs_x[1] = (1.5 * delta_x.y - 2.5) * delta_x.y * delta_x.y + 1;\n\
    cubic_coeffs_x[2] = (1.5 * delta_x.z - 2.5) * delta_x.z * delta_x.z + 1;\n\
    cubic_coeffs_x[3] = -0.5 * ((((delta_x.w - 5) * delta_x.w + 8) * delta_x.w) - 4);\n\
    delta_y.x = 1 + delta_y.y;\n\
    delta_y.z = 1 - delta_y.y;\n\
    delta_y.w = 2 - delta_y.y;\n\
    cubic_coeffs_y[0] = -0.5 * ((((delta_y.x - 5) * delta_y.x + 8) * delta_y.x) - 4);\n\
    cubic_coeffs_y[1] = (1.5 * delta_y.y - 2.5) * delta_y.y * delta_y.y + 1;\n\
    cubic_coeffs_y[2] = (1.5 * delta_y.z - 2.5) * delta_y.z * delta_y.z + 1;\n\
    cubic_coeffs_y[3] = -0.5 * ((((delta_y.w - 5) * delta_y.w + 8) * delta_y.w) - 4);\n\
    float dst = 0;\n\
    uint4 out = (uint4)(0,0,0,0);\n\
\n\
    data00   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    data10   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    data20   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    data30   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
\n\
    coord_in.y++;\n\
    data31   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x--;\n\
    data21   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x--;\n\
    data11   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x--;\n\
    data01   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
\n\
    coord_in.y++;\n\
    data02   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    data12   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    data22   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    data32   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
\n\
    coord_in.y++;\n\
    data33   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x--;\n\
    data23   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x--;\n\
    data13   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
    coord_in.x--;\n\
    data03   = convert_float(read_imageui(input, coord_in).x) * in_scale + in_tail;\n\
\n\
    dst = data00 * cubic_coeffs_x[0] * cubic_coeffs_y[0]\n\
        + data01 * cubic_coeffs_x[0] * cubic_coeffs_y[1]\n\
        + data02 * cubic_coeffs_x[0] * cubic_coeffs_y[2]\n\
        + data03 * cubic_coeffs_x[0] * cubic_coeffs_y[3]\n\
        + data10 * cubic_coeffs_x[1] * cubic_coeffs_y[0]\n\
        + data11 * cubic_coeffs_x[1] * cubic_coeffs_y[1]\n\
        + data12 * cubic_coeffs_x[1] * cubic_coeffs_y[2]\n\
        + data13 * cubic_coeffs_x[1] * cubic_coeffs_y[3]\n\
        + data20 * cubic_coeffs_x[2] * cubic_coeffs_y[0]\n\
        + data21 * cubic_coeffs_x[2] * cubic_coeffs_y[1]\n\
        + data22 * cubic_coeffs_x[2] * cubic_coeffs_y[2]\n\
        + data23 * cubic_coeffs_x[2] * cubic_coeffs_y[3]\n\
        + data30 * cubic_coeffs_x[3] * cubic_coeffs_y[0]\n\
        + data31 * cubic_coeffs_x[3] * cubic_coeffs_y[1]\n\
        + data32 * cubic_coeffs_x[3] * cubic_coeffs_y[2]\n\
        + data33 * cubic_coeffs_x[3] * cubic_coeffs_y[3];\n\
    out.x = convert_uint(dst * out_scale + out_tail);\n\
\n\
    write_imageui(output, coord_out, out);\n\
}\n\
"; /* end of resize_cubic_cl*/

static const char resize_nearest_cl[] = "\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    float  in_x       = (convert_float(coord_out.x) + half_pixel_value) * scale_x + round_value; \\\n\
    int    in_x_idx   = convert_int(in_x); \\\n\
    float  in_y       = (convert_float(coord_out.y) + half_pixel_value) * scale_y + round_value; \\\n\
    int    in_y_idx   = convert_int(in_y); \\\n\
\n\
__kernel void resize_nearest_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value,\n\
                           float  round_value)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, coord_out.z, 0);\n\
    float4 dst;\n\
    dst    = read_imagef(input, coord_in);\n\
    write_imagef(output, coord_out, dst);\n\
}\n\
\n\
\n\
__kernel void resize_nearest_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, coord_out.z, 0);\n\
    uint4 dst;\n\
    dst    = convert_uint4(convert_float4(read_imageui(input, coord_in)) * output_scale + output_tail);\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_nearest_cl*/

static const char reversesequence_cl[] = "#define REVERSESEQUENCE_axis2(name,src_type,readimage_type,\\\n\
                    convert_type,dst_type,writeimage_type) \\\n\
__kernel void reversesequence_##name( \\\n\
    __read_only  image2d_array_t  input0, \\\n\
    __read_only  image2d_t        input1, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inoutScale, \\\n\
                 float            inoutTail \\\n\
    ) \\\n\
{ \\\n\
    uint gidx = get_global_id(0); \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
\\\n\
    int4 coord_in = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_out = coord_in; \\\n\
    src_type src = readimage_type(input0, coord_in); \\\n\
    int src_index = read_imagei(input1, (int2)(gidz, 0)).x; \\\n\
    float4 src_temp = convert_float4(src); \\\n\
    dst_type dst = convert_type(src_temp * inoutScale + inoutTail); \\\n\
    if (gidy >= src_index) \\\n\
    { \\\n\
        writeimage_type(output, coord_out, dst); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        coord_out.y = src_index - 1 - coord_out.y; \\\n\
        writeimage_type(output, coord_out, dst); \\\n\
    } \\\n\
}\n\
REVERSESEQUENCE_axis2(F32toF32_axis2,float4,read_imagef,\\\n\
                      convert_float4,float4,write_imagef)\n\
REVERSESEQUENCE_axis2(F32toU32_axis2,float4,read_imagef,\\\n\
                      convert_uint4, uint4, write_imageui)\n\
REVERSESEQUENCE_axis2(F32toI32_axis2,float4,read_imagef,\\\n\
                       convert_int4,  int4,  write_imagei)\n\
REVERSESEQUENCE_axis2(I32toF32_axis2,int4,  read_imagei,\\\n\
                       convert_float4,float4,write_imagef)\n\
REVERSESEQUENCE_axis2(I32toU32_axis2,int4,  read_imagei,\\\n\
                     convert_uint4, uint4, write_imageui)\n\
REVERSESEQUENCE_axis2(I32toI32_axis2,int4,  read_imagei,\\\n\
                      convert_int4,  int4,  write_imagei)\n\
REVERSESEQUENCE_axis2(U32toF32_axis2,uint4, read_imageui,\\\n\
                      convert_float4,float4,write_imagef)\n\
REVERSESEQUENCE_axis2(U32toU32_axis2,uint4, read_imageui,\\\n\
                     convert_uint4, uint4, write_imageui)\n\
REVERSESEQUENCE_axis2(U32toI32_axis2,uint4, read_imageui,\\\n\
                        convert_int4,  int4,  write_imagei)\n\
\n\
__kernel void reversesequence_BF16toBF16_axis2(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            inoutScale,\n\
                 float            inoutTail\n\
    )\n\
{\n\
    uint gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    uint gidz = get_global_id(2);\n\
\n\
    int4 coord_in = (int4)(gidx, gidy, gidz, 0);\n\
    int4 coord_out = coord_in;\n\
    uint4 src = read_imageui(input0, coord_in);\n\
    int src_index = read_imagei(input1, (int2)(gidz, 0)).x;\n\
    uint4 dst = src;\n\
    if (gidy >= src_index)\n\
    {\n\
        write_imageui(output, coord_out, dst);\n\
    }\n\
    else\n\
    {\n\
        coord_out.y = src_index - 1 - coord_out.y;\n\
        write_imageui(output, coord_out, dst);\n\
    }\n\
}\n\
\n\
\n\
#define REVERSESEQUENCE_axis1(name,src_type,readimage_type,\\\n\
                             convert_type,dst_type,writeimage_type) \\\n\
__kernel void reversesequence_##name( \\\n\
    __read_only  image2d_array_t  input0, \\\n\
    __read_only  image2d_t        input1, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 float            inoutScale, \\\n\
                 float            inoutTail \\\n\
    ) \\\n\
{ \\\n\
    uint gidx = get_global_id(0); \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
\\\n\
    int4 coord_in = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_out = coord_in; \\\n\
    src_type src = readimage_type(input0, coord_in); \\\n\
    int src_index = read_imagei(input1, (int2)(gidy, 0)).x; \\\n\
    float4 src_temp = convert_float4(src); \\\n\
    dst_type dst = convert_type(src_temp * inoutScale + inoutTail ); \\\n\
    if (gidz >= src_index) \\\n\
    { \\\n\
        writeimage_type(output, coord_out, dst); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        coord_out.z = src_index - 1 - coord_out.z; \\\n\
        writeimage_type(output, coord_out, dst); \\\n\
    } \\\n\
}\n\
REVERSESEQUENCE_axis1(F32toF32_axis1,float4,read_imagef,\\\n\
                     convert_float4,float4,write_imagef)\n\
REVERSESEQUENCE_axis1(F32toU32_axis1,float4,read_imagef,\\\n\
                     convert_uint4, uint4, write_imageui)\n\
REVERSESEQUENCE_axis1(F32toI32_axis1,float4,read_imagef,\\\n\
                     convert_int4,  int4,  write_imagei)\n\
REVERSESEQUENCE_axis1(I32toF32_axis1,int4,  read_imagei,\\\n\
                     convert_float4,float4,write_imagef)\n\
REVERSESEQUENCE_axis1(I32toU32_axis1,int4,  read_imagei,\\\n\
                     convert_uint4, uint4, write_imageui)\n\
REVERSESEQUENCE_axis1(I32toI32_axis1,int4,  read_imagei,\\\n\
                     convert_int4,  int4,  write_imagei)\n\
REVERSESEQUENCE_axis1(U32toF32_axis1,uint4, read_imageui,\\\n\
                     convert_float4,float4,write_imagef)\n\
REVERSESEQUENCE_axis1(U32toU32_axis1,uint4, read_imageui,\\\n\
                     convert_uint4, uint4, write_imageui)\n\
REVERSESEQUENCE_axis1(U32toI32_axis1,uint4, read_imageui,\\\n\
                      convert_int4,  int4,  write_imagei)\n\
\n\
__kernel void reversesequence_BF16toBF16_axis1(\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            inoutScale,\n\
                 float            inoutTail\n\
    )\n\
{\n\
    uint gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    uint gidz = get_global_id(2);\n\
\n\
    int4 coord_in = (int4)(gidx, gidy, gidz, 0);\n\
    int4 coord_out = coord_in;\n\
    uint4 src = read_imageui(input0, coord_in);\n\
    int src_index = read_imagei(input1, (int2)(gidy, 0)).x;\n\
    uint4 dst = src;\n\
    if (gidz >= src_index)\n\
    {\n\
        write_imageui(output, coord_out, dst);\n\
    }\n\
    else\n\
    {\n\
        coord_out.z = src_index - 1 - coord_out.z;\n\
        write_imageui(output, coord_out, dst);\n\
    }\n\
}\n\
"; /* end of reversesequence_cl*/

static const char roi_align_cl[] = "#define VSI_NN_ROI_ALIGN_ANDROID 0\n\
\n\
inline float roi_align_1x1\n\
(\n\
    __read_only  image2d_array_t  input,\n\
                 float2 region_start,\n\
                 float2 region_end,\n\
                 float2 bin_size,\n\
                 int2   grid_size,\n\
                 float2 rcp_of_grid_size,\n\
                 int    pz,\n\
                 int4   max_spatial_dims,\n\
                 int    platform_type\n\
)\n\
{\n\
    float sum = 0;\n\
\n\
    for(int iy = 0; iy < grid_size.y; ++iy)\n\
    {\n\
        for(int ix = 0; ix < grid_size.x; ++ix)\n\
        {\n\
            float2 ixy = (float2)(ix + 0.5f, iy + 0.5f);\n\
            float2 pos = region_start + ixy * bin_size * rcp_of_grid_size;\n\
\n\
            int2 xy_low  = convert_int2(pos);\n\
            int2 xy_high = xy_low + 1;\n\
\n\
            if (VSI_NN_ROI_ALIGN_ANDROID == platform_type)\n\
            {\n\
                if (xy_low.x > max_spatial_dims.x || xy_low.x < -1 ||\n\
                    xy_low.y > max_spatial_dims.y || xy_low.y < -1 )\n\
                {\n\
                    continue;\n\
                }\n\
            }\n\
            else\n\
            {\n\
                if (pos.x > max_spatial_dims.x || pos.x < -1 ||\n\
                    pos.y > max_spatial_dims.y || pos.y < -1 )\n\
                {\n\
                    continue;\n\
                }\n\
            }\n\
\n\
            float2 lxy = pos - floor(pos);\n\
            float2 zero = 0;\n\
\n\
            lxy = xy_low >= max_spatial_dims.zw ? 0.0 : lxy;\n\
\n\
            float hy = 1.0f - lxy.y;\n\
            float hx = 1.0f - lxy.x;\n\
\n\
            float w1 = hy * hx;\n\
            float w2 = lxy.x - lxy.x * lxy.y;\n\
            float w3 = lxy.y - lxy.x * lxy.y;\n\
            float w4 = lxy.y * lxy.x;\n\
\n\
            float data1 = read_imagef(input, (int4)(xy_low.x, xy_low.y, pz, 0)).x;\n\
            float data2 = read_imagef(input, (int4)(xy_high.x, xy_low.y, pz, 0)).x;\n\
            float data3 = read_imagef(input, (int4)(xy_low.x, xy_high.y, pz, 0)).x;\n\
            float data4 = read_imagef(input, (int4)(xy_high.x, xy_high.y, pz, 0)).x;\n\
\n\
            sum = sum + w1 * data1 + w2 * data2 + w3 * data3 + w4 * data4;\n\
        }\n\
    }\n\
\n\
    return (float)(sum * rcp_of_grid_size.x * rcp_of_grid_size.y);\n\
}\n\
\n\
#define EPS_GRID 0.00001f\n\
#define TYPE_FLOAT16    (1)\n\
#define TYPE_FLOAT32    (2)\n\
__kernel void roi_align_F32_F32toF32\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       rois,\n\
    __read_only  image2d_t       n_rois,\n\
    __write_only image2d_array_t output,\n\
                 float           input_scale,\n\
                 float           input_tail,\n\
                 float           output_scale,\n\
                 float           output_zp,\n\
                 float           spatial_x_scale,\n\
                 float           spatial_y_scale,\n\
                 int             in_width,\n\
                 int             in_height,\n\
                 float           rcp_of_out_width,\n\
                 float           rcp_of_out_height,\n\
                 float           sampling_x_ratio,\n\
                 float           sampling_y_ratio,\n\
                 int             depth,\n\
                 int             dtype,\n\
                 int             platform_type\n\
)\n\
{\n\
    int px = get_global_id(0);\n\
    int py = get_global_id(1);\n\
    int pw = get_global_id(2);\n\
\n\
    int roi_batch = read_imagei(n_rois, (int2)(pw, 0)).x;\n\
    float4 roi_x = read_imagef(rois, (int2)(0, pw));\n\
    float4 roi_y = read_imagef(rois, (int2)(1, pw));\n\
    float4 roi_z = read_imagef(rois, (int2)(2, pw));\n\
    float4 roi_w = read_imagef(rois, (int2)(3, pw));\n\
    float4 roi = (float4)(roi_x.x, roi_y.x, roi_z.x, roi_w.x);\n\
\n\
    float4 roi_anchor = roi * (float4)(spatial_x_scale, spatial_y_scale, spatial_x_scale, spatial_y_scale);\n\
    float2 roi_dims = fmax(roi_anchor.zw - roi_anchor.xy, 1.0f);\n\
\n\
    float2 spatial_indx     = (float2)(px, py);\n\
    float2 pooled_dims      = (float2)(rcp_of_out_width, rcp_of_out_height);\n\
    int4 max_spatial_dims   = (int4)(in_width, in_height, in_width, in_height);\n\
    max_spatial_dims.zw = max_spatial_dims.zw - 1;\n\
\n\
    float2 max_limiatation = convert_float2(max_spatial_dims.zw);\n\
\n\
    float2 bin_size     = roi_dims * pooled_dims;\n\
    float2 region_start = spatial_indx * bin_size + roi_anchor.xy;\n\
    float2 region_end   = region_start + bin_size;\n\
\n\
    float2 roi_bin_grid = (float2)(sampling_x_ratio, sampling_y_ratio);\n\
\n\
    roi_bin_grid = roi_bin_grid == 0 ? ceil(bin_size - EPS_GRID) : roi_bin_grid;\n\
\n\
    int kz = roi_batch * depth;\n\
    float2 rcp_of_grid_size = 1.0f / roi_bin_grid;\n\
    int2 grid_size_xy = convert_int2(roi_bin_grid);\n\
    float4 interp;\n\
    int kz1 = pw * depth;\n\
    for (int pz = 0; pz < depth; pz ++, kz ++, kz1 ++)\n\
    {\n\
        interp.x = roi_align_1x1( input,\n\
                       region_start,\n\
                       region_end,\n\
                       bin_size,\n\
                       grid_size_xy,\n\
                       rcp_of_grid_size,\n\
                       kz,\n\
                       max_spatial_dims,\n\
                       platform_type);\n\
\n\
        if (dtype == TYPE_FLOAT16)\n\
        {\n\
            half tmp;\n\
            short dst;\n\
            _viv_asm(CONV, tmp, interp.x);\n\
            _viv_asm(COPY, dst, tmp, 2);\n\
\n\
            Tensor out_t =  create_tensor_from_image2d_array(output, 2);\n\
            short *output_ptr = (short *)get_tensor_ptr_from_coord(out_t, (int4)(px, py, kz1, 0));\n\
\n\
            output_ptr[0] = dst;\n\
        }\n\
        else\n\
        {\n\
            float4 dst = (float4)(interp.x,0,0,0);\n\
            int4 coord_dst = (int4)(px, py, kz1, 0);\n\
            write_imagef(output,coord_dst,dst);\n\
        }\n\
    }\n\
}\n\
\n\
inline float roi_align_1x1_U8toF32\n\
(\n\
    __read_only image2d_array_t  input,\n\
                float            input_scale,\n\
                float            input_tail,\n\
                float2           region_start,\n\
                float2           region_end,\n\
                float2           bin_size,\n\
                int2             grid_size,\n\
                float2           rcp_of_grid_size,\n\
                int              pz,\n\
                int4             max_spatial_dims,\n\
                int              platform_type\n\
)\n\
{\n\
    float sum = 0;\n\
\n\
    for(int iy = 0; iy < grid_size.y; ++iy)\n\
    {\n\
        for(int ix = 0; ix < grid_size.x; ++ix)\n\
        {\n\
            float2 ixy = (float2)(ix + 0.5f, iy + 0.5f);\n\
            float2 pos = region_start + ixy * bin_size * rcp_of_grid_size;\n\
\n\
            int2 xy_low  = convert_int2(pos);\n\
            int2 xy_high = xy_low + 1;\n\
\n\
            float2 lxy = pos - floor(pos);\n\
            float2 zero = 0;\n\
\n\
            if (VSI_NN_ROI_ALIGN_ANDROID == platform_type)\n\
            {\n\
                if (xy_low.x > max_spatial_dims.x || xy_low.x < -1 ||\n\
                    xy_low.y > max_spatial_dims.y || xy_low.y < -1 )\n\
                {\n\
                    continue;\n\
                }\n\
            }\n\
            else\n\
            {\n\
                if (pos.x > max_spatial_dims.x || pos.x < -1 ||\n\
                    pos.y > max_spatial_dims.y || pos.y < -1 )\n\
                {\n\
                    continue;\n\
                }\n\
            }\n\
\n\
            lxy = xy_low >= max_spatial_dims.zw ? 0.0 : lxy;\n\
\n\
            float hy = 1.0f - lxy.y;\n\
            float hx = 1.0f - lxy.x;\n\
\n\
            float w1 = hy * hx;\n\
            float w2 = lxy.x - lxy.x * lxy.y;\n\
            float w3 = lxy.y - lxy.x * lxy.y;\n\
            float w4 = lxy.y * lxy.x;\n\
\n\
            uint4 data;\n\
            data.x = read_imageui(input, (int4)(xy_low.x, xy_low.y, pz, 0)).x;\n\
            data.y = read_imageui(input, (int4)(xy_high.x, xy_low.y, pz, 0)).x;\n\
            data.z = read_imageui(input, (int4)(xy_low.x, xy_high.y, pz, 0)).x;\n\
            data.w = read_imageui(input, (int4)(xy_high.x, xy_high.y, pz, 0)).x;\n\
\n\
            float4 value = convert_float4(data) * input_scale + input_tail;\n\
\n\
            sum = sum + w1 * value.x + w2 * value.y + w3 * value.z + w4 * value.w;\n\
        }\n\
    }\n\
\n\
    return (float)(sum * rcp_of_grid_size.x * rcp_of_grid_size.y);\n\
\n\
}\n\
\n\
__kernel void roi_align_U8_U16toU8\n\
(\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       rois,\n\
    __read_only  image2d_t       n_rois,\n\
    __write_only image2d_array_t output,\n\
                 float           input_scale,\n\
                 float           input_tail,\n\
                 float           output_scale,\n\
                 float           output_zp,\n\
                 float           spatial_x_scale,\n\
                 float           spatial_y_scale,\n\
                 int             in_width,\n\
                 int             in_height,\n\
                 float           rcp_of_out_width,\n\
                 float           rcp_of_out_height,\n\
                 float           sampling_x_ratio,\n\
                 float           sampling_y_ratio,\n\
                 int             depth,\n\
                 int             dtype,\n\
                 int             platform_type\n\
)\n\
{\n\
    int px = get_global_id(0);\n\
    int py = get_global_id(1);\n\
    int pw = get_global_id(2);\n\
\n\
    int roi_batch = read_imagei(n_rois, (int2)(pw, 0)).x;\n\
    float4 roi_x = convert_float4(read_imageui(rois, (int2)(0, pw)));\n\
    float4 roi_y = convert_float4(read_imageui(rois, (int2)(1, pw)));\n\
    float4 roi_z = convert_float4(read_imageui(rois, (int2)(2, pw)));\n\
    float4 roi_w = convert_float4(read_imageui(rois, (int2)(3, pw)));\n\
    float4 roi = (float4)(roi_x.x, roi_y.x, roi_z.x, roi_w.x);\n\
\n\
    float4 roi_anchor = roi * (float4)(spatial_x_scale, spatial_y_scale, spatial_x_scale, spatial_y_scale);\n\
    float2 roi_dims = fmax(roi_anchor.zw - roi_anchor.xy, 1.0f);\n\
\n\
    float2 spatial_indx     = (float2)(px, py);\n\
    float2 pooled_dims      = (float2)(rcp_of_out_width, rcp_of_out_height);\n\
    int4 max_spatial_dims   = (int4)(in_width, in_height, in_width, in_height);\n\
    max_spatial_dims.zw = max_spatial_dims.zw - 1;\n\
\n\
    float2 max_limiatation = convert_float2(max_spatial_dims.zw);\n\
\n\
    float2 bin_size     = roi_dims * pooled_dims;\n\
    float2 region_start = spatial_indx * bin_size + roi_anchor.xy;\n\
    float2 region_end   = region_start + bin_size;\n\
\n\
    float2 roi_bin_grid = (float2)(sampling_x_ratio, sampling_y_ratio);\n\
\n\
    roi_bin_grid = roi_bin_grid == 0 ? ceil(bin_size - EPS_GRID) : roi_bin_grid;\n\
\n\
    int kz = roi_batch * depth;\n\
    float2 rcp_of_grid_size = 1.0f / roi_bin_grid;\n\
    int2 grid_size_xy = convert_int2(roi_bin_grid);\n\
    float4 interp;\n\
    int kz1 = pw * depth;\n\
    for (int pz = 0; pz < depth; pz ++, kz ++, kz1 ++)\n\
    {\n\
        interp.x = roi_align_1x1_U8toF32( input,\n\
                       input_scale,\n\
                       input_tail,\n\
                       region_start,\n\
                       region_end,\n\
                       bin_size,\n\
                       grid_size_xy,\n\
                       rcp_of_grid_size,\n\
                       kz,\n\
                       max_spatial_dims,\n\
                       platform_type);\n\
\n\
        uchar dst;\n\
        interp.x = interp.x * output_scale + output_zp;\n\
        interp.x = interp.x < 255 ? interp.x : 255;\n\
        dst = convert_uchar_rte(interp.x);\n\
\n\
        Tensor out_t =  create_tensor_from_image2d_array(output, 1);\n\
        uchar *output_ptr = (uchar *)get_tensor_ptr_from_coord(out_t, (int4)(px, py, kz1, 0));\n\
\n\
        output_ptr[0] = dst;\n\
    }\n\
}"; /* end of roi_align_cl*/

static const char rope_0_cl[] = "__kernel void rope_F32_F32toF32_axis0\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    float4 cos, sin;\n\
\n\
    READ_IMAGEF_2DARRAY(cos, cos_cache, coord);\n\
    READ_IMAGEF_2DARRAY(sin, sin_cache, coord);\n\
    coord.x = coord.x * step;\n\
    float4 src0 = read_imagef(input, coord);\n\
    int4 coord_out = coord;\n\
\n\
    coord.x += half_head_size;\n\
    float4 src1 = read_imagef(input, coord);\n\
\n\
    float4 dst0 = src0 * cos - src1 * sin;\n\
    float4 dst1 = src0 * sin + src1 * cos;\n\
\n\
    write_imagef(output, coord_out, dst0);\n\
    coord_out.x += half_head_size;\n\
    write_imagef(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_F32_F32toF32_axis1\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    float4 cos, sin;\n\
\n\
    READ_IMAGEF_2DARRAY(cos, cos_cache, coord);\n\
    READ_IMAGEF_2DARRAY(sin, sin_cache, coord);\n\
    coord.y = coord.y * step;\n\
    float4 src0 = read_imagef(input, coord);\n\
    int4 coord_out = coord;\n\
    coord.y += half_head_size;\n\
    float4 src1 = read_imagef(input, coord);\n\
\n\
    float4 dst0 = src0 * cos - src1 * sin;\n\
    float4 dst1 = src0 * sin + src1 * cos;\n\
\n\
    write_imagef(output, coord_out, dst0);\n\
    coord_out.y += half_head_size;\n\
    write_imagef(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_F32_F32toF32_axis2\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    float4 cos = read_imagef(cos_cache, coord);\n\
    float4 sin = read_imagef(sin_cache, coord);\n\
    coord.z = coord.z * step;\n\
    float4 src0 = read_imagef(input, coord);\n\
    int4 coord_out = coord;\n\
    coord.z += half_head_size;\n\
    float4 src1 = read_imagef(input, coord);\n\
\n\
    float4 dst0 = src0 * cos - src1 * sin;\n\
    float4 dst1 = src0 * sin + src1 * cos;\n\
\n\
    write_imagef(output, coord_out, dst0);\n\
    coord_out.z += half_head_size;\n\
    write_imagef(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_I32_I32toI32_axis0\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 _cos, _sin;\n\
    float4 cos, sin;\n\
\n\
    READ_IMAGEI_2DARRAY(_cos, cos_cache, coord);\n\
    READ_IMAGEI_2DARRAY(_sin, sin_cache, coord);\n\
    coord.x = coord.x * step;\n\
    float4 src0 = convert_float4(read_imagei(input, coord));\n\
    int4 coord_out = coord;\n\
\n\
    coord.x += half_head_size;\n\
    float4 src1 = convert_float4(read_imagei(input, coord));\n\
\n\
    src0 = src0 - input_zp;\n\
    src1 = src1 - input_zp;\n\
    cos = convert_float4(_cos) - cos_zp;\n\
    sin = convert_float4(_sin) - sin_zp;\n\
\n\
    float4 _dst0 = src0 * cos * scale0 - src1 * sin * scale1 + output_zp;\n\
    float4 _dst1 = src0 * sin * scale1 + src1 * cos * scale0 + output_zp;\n\
    int4 dst0 = convert_int4_rte(_dst0);\n\
    int4 dst1 = convert_int4_rte(_dst1);\n\
\n\
    write_imagei(output, coord_out, dst0);\n\
    coord_out.x += half_head_size;\n\
    write_imagei(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_I32_I32toI32_axis1\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 _cos, _sin;\n\
    float4 cos, sin;\n\
\n\
    READ_IMAGEI_2DARRAY(_cos, cos_cache, coord);\n\
    READ_IMAGEI_2DARRAY(_sin, sin_cache, coord);\n\
    coord.y = coord.y * step;\n\
    float4 src0 = convert_float4(read_imagei(input, coord));\n\
    int4 coord_out = coord;\n\
\n\
    coord.y += half_head_size;\n\
    float4 src1 = convert_float4(read_imagei(input, coord));\n\
\n\
    src0 = src0 - input_zp;\n\
    src1 = src1 - input_zp;\n\
    cos = convert_float4(_cos) - cos_zp;\n\
    sin = convert_float4(_sin) - sin_zp;\n\
\n\
    float4 _dst0 = src0 * cos * scale0 - src1 * sin * scale1 + output_zp;\n\
    float4 _dst1 = src0 * sin * scale1 + src1 * cos * scale0 + output_zp;\n\
    int4 dst0 = convert_int4_rte(_dst0);\n\
    int4 dst1 = convert_int4_rte(_dst1);\n\
\n\
    write_imagei(output, coord_out, dst0);\n\
    coord_out.y += half_head_size;\n\
    write_imagei(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_I32_I32toI32_axis2\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    float4 cos = convert_float4(read_imagei(cos_cache, coord));\n\
    float4 sin = convert_float4(read_imagei(sin_cache, coord));\n\
    coord.z = coord.z * step;\n\
    float4 src0 = convert_float4(read_imagei(input, coord));\n\
    int4 coord_out = coord;\n\
\n\
    coord.z += half_head_size;\n\
    float4 src1 = convert_float4(read_imagei(input, coord));\n\
\n\
    src0 = src0 - input_zp;\n\
    src1 = src1 - input_zp;\n\
    cos = cos - cos_zp;\n\
    sin = sin - sin_zp;\n\
\n\
    float4 _dst0 = src0 * cos * scale0 - src1 * sin * scale1 + output_zp;\n\
    float4 _dst1 = src0 * sin * scale1 + src1 * cos * scale0 + output_zp;\n\
    int4 dst0 = convert_int4_rte(_dst0);\n\
    int4 dst1 = convert_int4_rte(_dst1);\n\
\n\
    write_imagei(output, coord_out, dst0);\n\
    coord_out.z += half_head_size;\n\
    write_imagei(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_U32_U32toU32_axis0\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    uint4 _cos, _sin;\n\
    float4 cos, sin;\n\
\n\
    READ_IMAGEUI_2DARRAY(_cos, cos_cache, coord);\n\
    READ_IMAGEUI_2DARRAY(_sin, sin_cache, coord);\n\
    coord.x = coord.x * step;\n\
    float4 src0 = convert_float4(read_imageui(input, coord));\n\
    int4 coord_out = coord;\n\
\n\
    coord.x += half_head_size;\n\
    float4 src1 = convert_float4(read_imageui(input, coord));\n\
\n\
    src0 = src0 - input_zp;\n\
    src1 = src1 - input_zp;\n\
    cos = convert_float4(_cos) - cos_zp;\n\
    sin = convert_float4(_sin) - sin_zp;\n\
\n\
    float4 _dst0 = src0 * cos * scale0 - src1 * sin * scale1 + output_zp;\n\
    float4 _dst1 = src0 * sin * scale1 + src1 * cos * scale0 + output_zp;\n\
    uint4 dst0 = convert_uint4_rte(_dst0);\n\
    uint4 dst1 = convert_uint4_rte(_dst1);\n\
\n\
    write_imageui(output, coord_out, dst0);\n\
    coord_out.x += half_head_size;\n\
    write_imageui(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_U32_U32toU32_axis1\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    uint4 _cos, _sin;\n\
    float4 cos, sin;\n\
\n\
    READ_IMAGEUI_2DARRAY(_cos, cos_cache, coord);\n\
    READ_IMAGEUI_2DARRAY(_sin, sin_cache, coord);\n\
    coord.y = coord.y * step;\n\
    float4 src0 = convert_float4(read_imageui(input, coord));\n\
    int4 coord_out = coord;\n\
\n\
    coord.y += half_head_size;\n\
    float4 src1 = convert_float4(read_imageui(input, coord));\n\
\n\
    src0 = src0 - input_zp;\n\
    src1 = src1 - input_zp;\n\
    cos = convert_float4(_cos) - cos_zp;\n\
    sin = convert_float4(_sin) - sin_zp;\n\
\n\
    float4 _dst0 = src0 * cos * scale0 - src1 * sin * scale1 + output_zp;\n\
    float4 _dst1 = src0 * sin * scale1 + src1 * cos * scale0 + output_zp;\n\
    uint4 dst0 = convert_uint4_rte(_dst0);\n\
    uint4 dst1 = convert_uint4_rte(_dst1);\n\
\n\
    write_imageui(output, coord_out, dst0);\n\
    coord_out.y += half_head_size;\n\
    write_imageui(output, coord_out, dst1);\n\
}\n\
\n\
__kernel void rope_U32_U32toU32_axis2\n\
  (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t cos_cache,\n\
    __read_only  image2d_array_t sin_cache,\n\
    __write_only image2d_array_t output,\n\
                 int axis,\n\
                 float input_zp,\n\
                 float cos_zp,\n\
                 float sin_zp,\n\
                 float scale0,\n\
                 float scale1,\n\
                 float output_zp,\n\
                 int half_head_size,\n\
                 int step\n\
  )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    float4 cos = convert_float4(read_imageui(cos_cache, coord));\n\
    float4 sin = convert_float4(read_imageui(sin_cache, coord));\n\
    coord.z = coord.z * step;\n\
    float4 src0 = convert_float4(read_imageui(input, coord));\n\
    int4 coord_out = coord;\n\
\n\
    coord.z += half_head_size;\n\
    float4 src1 = convert_float4(read_imageui(input, coord));\n\
\n\
    src0 = src0 - input_zp;\n\
    src1 = src1 - input_zp;\n\
    cos = cos - cos_zp;\n\
    sin = sin - sin_zp;\n\
\n\
    float4 _dst0 = src0 * cos * scale0 - src1 * sin * scale1 + output_zp;\n\
    float4 _dst1 = src0 * sin * scale1 + src1 * cos * scale0 + output_zp;\n\
    uint4 dst0 = convert_uint4_rte(_dst0);\n\
    uint4 dst1 = convert_uint4_rte(_dst1);\n\
\n\
    write_imageui(output, coord_out, dst0);\n\
    coord_out.z += half_head_size;\n\
    write_imageui(output, coord_out, dst1);\n\
}\n\
"; /* end of rope_0_cl*/

static const char scatter_elements_cl[] = "\n\
#define SCATTER_ELEMENTS_AXIS0_32BITS_IMPL(name, dtype) \\\n\
__kernel void scatter_elements_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 4); \\\n\
    Image update_i = create_image_from_image2d(update, 4); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 4); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = ref_ptr[0]; \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data = update_ptr[x]; \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SCATTER_ELEMENTS_AXIS0_32BITS_IMPL(F32_I32_F32toF32, float)\n\
SCATTER_ELEMENTS_AXIS0_32BITS_IMPL(I32_I32_I32toI32, int)\n\
\n\
#define SCATTER_ELEMENTS_AXIS0_16BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 2); \\\n\
    Image update_i = create_image_from_image2d(update, 2); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data = conv_func(convert_float(update_ptr[x]) * update_scale + update_tail + output_zp); \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SCATTER_ELEMENTS_AXIS0_16BITS_IMPL(I16_I32_I16toI16,    short,  convert_short_rte)\n\
SCATTER_ELEMENTS_AXIS0_16BITS_IMPL(F16_I32_F16toF16,    short,  convert_short)\n\
SCATTER_ELEMENTS_AXIS0_16BITS_IMPL(BF16_I32_BF16toBF16, ushort, convert_ushort)\n\
\n\
#define SCATTER_ELEMENTS_AXIS0_8BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 1); \\\n\
    Image update_i = create_image_from_image2d(update, 1); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 1); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data = conv_func(convert_float(update_ptr[x]) * update_scale + update_tail + output_zp); \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SCATTER_ELEMENTS_AXIS0_8BITS_IMPL(U8_I32_U8toU8, uchar, convert_uchar_rte)\n\
SCATTER_ELEMENTS_AXIS0_8BITS_IMPL(I8_I32_I8toI8, char,  convert_char)\n\
\n\
#define SCATTER_ELEMENTS_AXIS1_32BITS_IMPL(name, dtype) \\\n\
__kernel void scatter_elements_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 4); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 4); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 4); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = ref_ptr[0]; \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data = update_ptr[y * inner_size]; \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SCATTER_ELEMENTS_AXIS1_32BITS_IMPL(F32_I32_F32toF32, float)\n\
SCATTER_ELEMENTS_AXIS1_32BITS_IMPL(I32_I32_I32toI32, int)\n\
\n\
#define SCATTER_ELEMENTS_AXIS1_16BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 2); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 2); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 2); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data = conv_func(convert_float(update_ptr[y * inner_size]) \\\n\
                            * update_scale + update_tail + output_zp); \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SCATTER_ELEMENTS_AXIS1_16BITS_IMPL(I16_I32_I16toI16,    short,  convert_short_rte)\n\
SCATTER_ELEMENTS_AXIS1_16BITS_IMPL(F16_I32_F16toF16,    short,  convert_short)\n\
SCATTER_ELEMENTS_AXIS1_16BITS_IMPL(BF16_I32_BF16toBF16, ushort, convert_ushort)\n\
\n\
#define SCATTER_ELEMENTS_AXIS1_8BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 1); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 1); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 1); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data = conv_func(convert_float(update_ptr[y * inner_size]) \\\n\
                                * update_scale + update_tail + output_zp); \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SCATTER_ELEMENTS_AXIS1_8BITS_IMPL(U8_I32_U8toU8, uchar, convert_uchar_rte)\n\
SCATTER_ELEMENTS_AXIS1_8BITS_IMPL(I8_I32_I8toI8, char,  convert_char)\n\
"; /* end of scatter_elements_cl*/

static const char scatter_elements_add_cl[] = "\n\
#define SE_ADD_AXIS0_32BITS_IMPL(name, dtype) \\\n\
__kernel void scatter_elements_add_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 4); \\\n\
    Image update_i = create_image_from_image2d(update, 4); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 4); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = ref_ptr[0]; \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data += update_ptr[x]; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_ADD_AXIS0_32BITS_IMPL(F32_I32_F32toF32, float)\n\
SE_ADD_AXIS0_32BITS_IMPL(I32_I32_I32toI32, int)\n\
\n\
#define SE_ADD_AXIS0_16BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_add_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 2); \\\n\
    Image update_i = create_image_from_image2d(update, 2); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data += conv_func(convert_float(update_ptr[x]) * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_ADD_AXIS0_16BITS_IMPL(I16_I32_I16toI16,    short,  convert_short_rte)\n\
SE_ADD_AXIS0_16BITS_IMPL(F16_I32_F16toF16,    short,  convert_short)\n\
SE_ADD_AXIS0_16BITS_IMPL(BF16_I32_BF16toBF16, ushort, convert_ushort)\n\
\n\
#define SE_ADD_AXIS0_8BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_add_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 1); \\\n\
    Image update_i = create_image_from_image2d(update, 1); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 1); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data += conv_func(convert_float(update_ptr[x]) * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_ADD_AXIS0_8BITS_IMPL(U8_I32_U8toU8, uchar, convert_uchar_rte)\n\
SE_ADD_AXIS0_8BITS_IMPL(I8_I32_I8toI8, char,  convert_char)\n\
\n\
#define SE_ADD_AXIS1_32BITS_IMPL(name, dtype) \\\n\
__kernel void scatter_elements_add_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 4); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 4); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 4); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = ref_ptr[0]; \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data += update_ptr[y * inner_size]; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_ADD_AXIS1_32BITS_IMPL(F32_I32_F32toF32, float)\n\
SE_ADD_AXIS1_32BITS_IMPL(I32_I32_I32toI32, int)\n\
\n\
#define SE_ADD_AXIS1_16BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_add_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 2); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 2); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 2); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data += conv_func(convert_float(update_ptr[y * inner_size]) \\\n\
                            * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_ADD_AXIS1_16BITS_IMPL(I16_I32_I16toI16,    short,  convert_short_rte)\n\
SE_ADD_AXIS1_16BITS_IMPL(F16_I32_F16toF16,    short,  convert_short)\n\
SE_ADD_AXIS1_16BITS_IMPL(BF16_I32_BF16toBF16, ushort, convert_ushort)\n\
\n\
#define SE_ADD_AXIS1_8BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_add_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 1); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 1); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 1); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data += conv_func(convert_float(update_ptr[y * inner_size]) \\\n\
                                * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_ADD_AXIS1_8BITS_IMPL(U8_I32_U8toU8, uchar, convert_uchar_rte)\n\
SE_ADD_AXIS1_8BITS_IMPL(I8_I32_I8toI8, char,  convert_char)\n\
"; /* end of scatter_elements_add_cl*/

static const char scatter_elements_mul_cl[] = "\n\
#define SE_MUL_AXIS0_32BITS_IMPL(name, dtype) \\\n\
__kernel void scatter_elements_mul_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 4); \\\n\
    Image update_i = create_image_from_image2d(update, 4); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 4); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = ref_ptr[0]; \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data *= update_ptr[x]; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_MUL_AXIS0_32BITS_IMPL(F32_I32_F32toF32, float)\n\
SE_MUL_AXIS0_32BITS_IMPL(I32_I32_I32toI32, int)\n\
\n\
#define SE_MUL_AXIS0_16BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_mul_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 2); \\\n\
    Image update_i = create_image_from_image2d(update, 2); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data *= conv_func(convert_float(update_ptr[x]) * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_MUL_AXIS0_16BITS_IMPL(I16_I32_I16toI16,    short,  convert_short_rte)\n\
SE_MUL_AXIS0_16BITS_IMPL(F16_I32_F16toF16,    short,  convert_short)\n\
SE_MUL_AXIS0_16BITS_IMPL(BF16_I32_BF16toBF16, ushort, convert_ushort)\n\
\n\
#define SE_MUL_AXIS0_8BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_mul_axis0_##name \\\n\
    ( \\\n\
    __read_only  image2d_t ref, \\\n\
    __read_only  image2d_t indices, \\\n\
    __read_only  image2d_t update, \\\n\
    __write_only image2d_t output, \\\n\
                 int       axis, \\\n\
                 int       reduction, \\\n\
                 float     ref_scale, \\\n\
                 float     ref_tail, \\\n\
                 float     update_scale, \\\n\
                 float     update_tail, \\\n\
                 float     output_zp, \\\n\
                 int       inner_size, \\\n\
                 int       axis_size, \\\n\
                 int       outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
 \\\n\
    Image ref_i = create_image_from_image2d(ref, 1); \\\n\
    Image update_i = create_image_from_image2d(update, 1); \\\n\
    Image indices_i = create_image_from_image2d(indices, 4); \\\n\
    Image output_i = create_image_from_image2d(output, 1); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_image_ptr_from_coord(ref_i, coord.xy); \\\n\
    dtype *output_ptr = (dtype *)get_image_ptr_from_coord(output_i, coord.xy); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.y < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_image_ptr_from_coord(update_i, coord.wy); \\\n\
        int *indices_ptr = (int *)get_image_ptr_from_coord(indices_i, coord.wy); \\\n\
        for(int x = 0; x < axis_size; x ++) \\\n\
        { \\\n\
            int offset = indices_ptr[x]; \\\n\
            if (offset == coord.x) \\\n\
            { \\\n\
                data *= conv_func(convert_float(update_ptr[x]) * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_MUL_AXIS0_8BITS_IMPL(U8_I32_U8toU8, uchar, convert_uchar_rte)\n\
SE_MUL_AXIS0_8BITS_IMPL(I8_I32_I8toI8, char,  convert_char)\n\
\n\
#define SE_MUL_AXIS1_32BITS_IMPL(name, dtype) \\\n\
__kernel void scatter_elements_mul_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 4); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 4); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 4); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = ref_ptr[0]; \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data *= update_ptr[y * inner_size]; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_MUL_AXIS1_32BITS_IMPL(F32_I32_F32toF32, float)\n\
SE_MUL_AXIS1_32BITS_IMPL(I32_I32_I32toI32, int)\n\
\n\
#define SE_MUL_AXIS1_16BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_mul_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 2); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 2); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 2); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data *= conv_func(convert_float(update_ptr[y * inner_size]) \\\n\
                            * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_MUL_AXIS1_16BITS_IMPL(I16_I32_I16toI16,    short,  convert_short_rte)\n\
SE_MUL_AXIS1_16BITS_IMPL(F16_I32_F16toF16,    short,  convert_short)\n\
SE_MUL_AXIS1_16BITS_IMPL(BF16_I32_BF16toBF16, ushort, convert_ushort)\n\
\n\
#define SE_MUL_AXIS1_8BITS_IMPL(name, dtype, conv_func) \\\n\
__kernel void scatter_elements_mul_axis1_##name \\\n\
    ( \\\n\
    __read_only  image2d_array_t ref, \\\n\
    __read_only  image2d_array_t indices, \\\n\
    __read_only  image2d_array_t update, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             axis, \\\n\
                 int             reduction, \\\n\
                 float           ref_scale, \\\n\
                 float           ref_tail, \\\n\
                 float           update_scale, \\\n\
                 float           update_tail, \\\n\
                 float           output_zp, \\\n\
                 int             inner_size, \\\n\
                 int             axis_size, \\\n\
                 int             outer_size \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    Tensor ref_i = create_tensor_from_image2d_array(ref, 1); \\\n\
    Tensor update_i = create_tensor_from_image2d_array(update, 1); \\\n\
    Tensor indices_i = create_tensor_from_image2d_array(indices, 4); \\\n\
    Tensor output_i = create_tensor_from_image2d_array(output, 1); \\\n\
 \\\n\
    dtype *ref_ptr = (dtype *)get_tensor_ptr_from_coord(ref_i, coord); \\\n\
    dtype *output_ptr = (dtype *)get_tensor_ptr_from_coord(output_i, coord); \\\n\
    dtype data = conv_func(convert_float(ref_ptr[0]) * ref_scale + ref_tail + output_zp); \\\n\
    if (coord.x < inner_size && coord.z < outer_size) \\\n\
    { \\\n\
        dtype *update_ptr = (dtype *)get_tensor_ptr_from_coord(update_i, coord.xwzw); \\\n\
        int *indices_ptr = (int *)get_tensor_ptr_from_coord(indices_i, coord.xwzw); \\\n\
        for(int y = 0; y < axis_size; y ++) \\\n\
        { \\\n\
            int offset = indices_ptr[y * inner_size]; \\\n\
            if (offset == coord.y) \\\n\
            { \\\n\
                data *= conv_func(convert_float(update_ptr[y * inner_size]) \\\n\
                                * update_scale + update_tail + output_zp); \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    output_ptr[0] = data; \\\n\
}\n\
SE_MUL_AXIS1_8BITS_IMPL(U8_I32_U8toU8, uchar, convert_uchar_rte)\n\
SE_MUL_AXIS1_8BITS_IMPL(I8_I32_I8toI8, char,  convert_char)\n\
"; /* end of scatter_elements_mul_cl*/

static const char scatter_nd_cl[] = "__kernel void scatter_nd_U32toU32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    uint4 sum = (uint4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        if(gidy == indice.x)\n\
        {\n\
            uint4 data = read_imageui(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imageui(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_U32toU32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    uint4 sum = (uint4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int idx = indice0.x * width + indice1.x;\n\
        if(gidy == idx)\n\
        {\n\
            uint4 data = read_imageui(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imageui(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_U32toU32_3D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    uint4 sum = (uint4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int4 indice2 = read_imagei(input0, (int2)(2, i));\n\
        int idx = indice0.x * area + indice1.x * width + indice2.x;\n\
        if(gidy == idx)\n\
        {\n\
            uint4 data = read_imageui(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imageui(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_I32toI32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 sum = (int4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        if(gidy == indice.x)\n\
        {\n\
            int4 data = read_imagei(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagei(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_I32toI32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 sum = (int4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int idx = indice0.x * width + indice1.x;\n\
        if(gidy == idx)\n\
        {\n\
            int4 data = read_imagei(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagei(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_I32toI32_3D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 sum = (int4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int4 indice2 = read_imagei(input0, (int2)(2, i));\n\
        int idx = indice0.x * area + indice1.x * width + indice2.x;\n\
        if(gidy == idx)\n\
        {\n\
            int4 data = read_imagei(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagei(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_F32toF32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        if(gidy == indice.x)\n\
        {\n\
            float4 data = read_imagef(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagef(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_F32toF32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int idx = indice0.x * width + indice1.x;\n\
        if(gidy == idx)\n\
        {\n\
            float4 data = read_imagef(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagef(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_F32toF32_3D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int4 indice2 = read_imagei(input0, (int2)(2, i));\n\
        int idx = indice0.x * area + indice1.x * width + indice2.x;\n\
        if(gidy == idx)\n\
        {\n\
            float4 data = read_imagef(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagef(output, (int2)(gidx, gidy), sum);\n\
}"; /* end of scatter_nd_cl*/

static const char scatter_nd_update_cl[] = "\n\
#define SCATTER_ND_UPDATE(src0_type, data_type, read_func, write_func) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##to##src0_type( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    __write_only image2d_t  output, \\\n\
    int offsetX, \\\n\
    int offsetY, \\\n\
    int offsetZ, \\\n\
    int offsetW, \\\n\
    int offset_idx, \\\n\
    int coord_dim, \\\n\
    int index_num \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int cnt = 0; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + offset_idx); \\\n\
        index_ptr += coord_dim; \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type data = read_func(input2, (int2)(gidx, i)); \\\n\
            cnt++; \\\n\
            sum += data; \\\n\
        } \\\n\
    } \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        sum = read_func(input0, coord); \\\n\
    } \\\n\
    write_func(output, coord, sum); \\\n\
}\n\
SCATTER_ND_UPDATE(U32,  uint4,  read_imageui, write_imageui)\n\
SCATTER_ND_UPDATE(I32,  int4,   read_imagei,  write_imagei)\n\
SCATTER_ND_UPDATE(F32,  float4, read_imagef,  write_imagef)\n\
"; /* end of scatter_nd_update_cl*/

static const char scatter_nd_update_reduction_cl[] = "\n\
inline void AtomicAdd_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = prevVal.floatVal + operand;\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
inline void AtomicMul_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = prevVal.floatVal * operand;\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
inline void AtomicMax_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = fmax(prevVal.floatVal, operand);\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
inline void AtomicMin_float(volatile __global float *source, const float operand)\n\
{\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } newVal;\n\
    union\n\
    {\n\
        unsigned int intVal;\n\
        float floatVal;\n\
    } prevVal;\n\
    do\n\
    {\n\
        prevVal.floatVal = *source;\n\
        newVal.floatVal = fmin(prevVal.floatVal, operand);\n\
    } while(atomic_cmpxchg((volatile __global unsigned int *)source,\n\
                             prevVal.intVal, newVal.intVal) != prevVal.intVal);\n\
}\n\
\n\
#define SCATTER_REDUCTION_PREPROCESS(name0, ptr0, type0, size0, ptr2) \\\n\
__kernel void scatter_nd_update_reduction_preprocess_##name0( \\\n\
    __read_only image2d_t   input_ref, \\\n\
    image2d_t  temp_buf_float, \\\n\
    int length, int res, float input_scale, float zp_scale) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img1 = create_image_from_image2d(input_ref, size0); \\\n\
    Image img2 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    __global float* tmp_ref_ptr = (__global float*)img2.ptr; \\\n\
    type0 src0, src1; \\\n\
    float4 tmpDst0, tmpDst1; \\\n\
    __global ptr2* input_ptr = (__global ptr2*)img1.ptr; \\\n\
    if(length > 0) \\\n\
    { \\\n\
        int loc2 = gidx * 8; \\\n\
        ptr0 tmpData0 = vload4(0, input_ptr + loc2); \\\n\
        ptr0 tmpData1 = vload4(1, input_ptr + loc2); \\\n\
        _viv_asm(COPY, src0, tmpData0, 16); \\\n\
        _viv_asm(COPY, src1, tmpData1, 16); \\\n\
        _viv_asm(CONV, tmpDst0, src0); \\\n\
        _viv_asm(CONV, tmpDst1, src1); \\\n\
        tmpDst0 = tmpDst0 * input_scale + zp_scale; \\\n\
        tmpDst1 = tmpDst1 * input_scale + zp_scale; \\\n\
        vstore4(tmpDst0, 0, tmp_ref_ptr + loc2); \\\n\
        vstore4(tmpDst1, 1, tmp_ref_ptr + loc2); \\\n\
    } \\\n\
    for(int i = gidx; i < res; i += get_global_size(0)) \\\n\
    { \\\n\
        ptr2 tmpData0 = input_ptr[length + i]; \\\n\
        _viv_asm(COPY, src0, tmpData0, 4); \\\n\
        _viv_asm(CONV, tmpDst0, src0); \\\n\
        tmpDst0.x = tmpDst0.x * input_scale + zp_scale; \\\n\
        tmp_ref_ptr[length + i] = tmpDst0.x; \\\n\
    } \\\n\
}\n\
SCATTER_REDUCTION_PREPROCESS(U8,  uchar4, uchar4, 1, uchar)\n\
SCATTER_REDUCTION_PREPROCESS(I8,  char4,  char4,  1, char)\n\
SCATTER_REDUCTION_PREPROCESS(I16, short4, short4, 2, short)\n\
SCATTER_REDUCTION_PREPROCESS(F16, short4, half4,  2, short)\n\
SCATTER_REDUCTION_PREPROCESS(F32, float4, float4, 4, float)\n\
\n\
#define SCATTER_ND_REDUCTION_PROCESS_F16(name0, func) \\\n\
__kernel void scatter_nd_update_reduction_##name0##_F16( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    image2d_t  temp_buf_float, \\\n\
    image2d_t  link_buffer0, \\\n\
    int val0, int val1, int val2, int val3, int val4, int val5, int val6, \\\n\
    int coord_dim, int update_width, int output_width, float update_scale, float zp_scale) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(update, 2); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global short* update_ptr = (__global short*)img2.ptr; \\\n\
    __global float* output_ptr = (__global float*)img3.ptr; \\\n\
    half src; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    short tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    int idx = indice.x * val0 + indice.y * val1 + indice.z * val2 + indice.w * val3; \\\n\
    idx = idx + indice1.x * val4 + indice1.y * val5 + indice1.z * val6; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    _viv_asm(COPY, src, tmpData, 4); \\\n\
    float data; \\\n\
    _viv_asm(CONV, data, src); \\\n\
    func(output_ptr + loc, data); \\\n\
}\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Add,  AtomicAdd_float)\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Mul,  AtomicMul_float)\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Max,  AtomicMax_float)\n\
SCATTER_ND_REDUCTION_PROCESS_F16(Min,  AtomicMin_float)\n\
\n\
#define SCATTER_ND_UPDATE_PROCESS_QINT(name0, src0_type, ptr_type, element_size, func) \\\n\
__kernel void scatter_nd_update_reduction_##name0##_##src0_type( \\\n\
    __read_only image2d_t   index, \\\n\
    __read_only image2d_t   update, \\\n\
    image2d_t  temp_buf_float, \\\n\
    image2d_t  link_buffer0, \\\n\
    int val0, int val1, int val2, int val3, int val4, int val5, int val6, \\\n\
    int coord_dim, int update_width, int output_width, float update_scale, float zp_scale) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img1 = create_image_from_image2d(index, 4); \\\n\
    Image img2 = create_image_from_image2d(update, element_size); \\\n\
    Image img3 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global float* output_ptr = (__global float*)img3.ptr; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim); \\\n\
    int4 indice1 = coord_dim < 5 ? (int4)(0) : vload4(1, index_ptr + gidy * coord_dim); \\\n\
    ptr_type tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    int idx = indice.x * val0 + indice.y * val1 + indice.z * val2 + indice.w * val3; \\\n\
    idx = idx + indice1.x * val4 + indice1.y * val5 + indice1.z * val6; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    float data; \\\n\
    _viv_asm(CONV, data, tmpData); \\\n\
    data = data * update_scale + zp_scale; \\\n\
    func(output_ptr + loc, data); \\\n\
}\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Add, U8,  uchar, 1, AtomicAdd_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Mul, U8,  uchar, 1, AtomicMul_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Max, U8,  uchar, 1, AtomicMax_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Min, U8,  uchar, 1, AtomicMin_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Add, I8,  char,  1, AtomicAdd_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Mul, I8,  char,  1, AtomicMul_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Max, I8,  char,  1, AtomicMax_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Min, I8,  char,  1, AtomicMin_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Add, I16, short, 2, AtomicAdd_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Mul, I16, short, 2, AtomicMul_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Max, I16, short, 2, AtomicMax_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Min, I16, short, 2, AtomicMin_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Add, F32, float, 4, AtomicAdd_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Mul, F32, float, 4, AtomicMul_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Max, F32, float, 4, AtomicMax_float)\n\
SCATTER_ND_UPDATE_PROCESS_QINT(Min, F32, float, 4, AtomicMin_float)"; /* end of scatter_nd_update_reduction_cl*/

static const char scatter_nd_update_reduction_conv_cl[] = "__kernel void scatter_nd_update_reduction_conv_F16(\n\
    __read_only image2d_t  temp_buf_float,\n\
    __read_only image2d_t  link_buf,\n\
    image2d_t  output,\n\
    int length, int res, float output_scale, float output_zp)\n\
{\n\
    int gidx = get_global_id(0);\n\
    Image img1 = create_image_from_image2d(temp_buf_float, 4);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    __global float* input_ptr = (__global float*)img1.ptr;\n\
    __global short* output_ptr = (__global short*)img2.ptr;\n\
    if(length > 0)\n\
    {\n\
        int offset = gidx * 8;\n\
        float4 src0 = vload4(0, input_ptr + offset);\n\
        float4 src1 = vload4(1, input_ptr + offset);\n\
        half4 data0, data1;\n\
        _viv_asm(CONV, data0, src0);\n\
        _viv_asm(CONV, data1, src1);\n\
        short4 dst0, dst1;\n\
        _viv_asm(COPY, dst0, data0, 16);\n\
        _viv_asm(COPY, dst1, data1, 16);\n\
        vstore4(dst0, 0, output_ptr + offset);\n\
        vstore4(dst1, 1, output_ptr + offset);\n\
    }\n\
    for(int i = gidx; i < res; i += get_global_size(0))\n\
    {\n\
        float src = input_ptr[length + i];\n\
        half data;\n\
        _viv_asm(CONV, data, src);\n\
        short dst;\n\
        _viv_asm(COPY, dst, data, 4);\n\
        output_ptr[length + i] = dst;\n\
    }\n\
}\n\
\n\
#define SCATTER_ND_UPDATE_CONV(src0_type, ptr_type, element_size, ptr_type1, conv_func) \\\n\
__kernel void scatter_nd_update_reduction_conv_##src0_type( \\\n\
    __read_only image2d_t  temp_buf_float, \\\n\
    __read_only image2d_t  link_buf, \\\n\
    image2d_t  output, \\\n\
    int length, int res, float output_scale, float output_zp) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    Image img1 = create_image_from_image2d(temp_buf_float, 4); \\\n\
    Image img2 = create_image_from_image2d(output, element_size); \\\n\
    __global float* input_ptr = (__global float*)img1.ptr; \\\n\
    __global ptr_type1* output_ptr = (__global ptr_type1*)img2.ptr; \\\n\
    if(length > 0) \\\n\
    { \\\n\
        int offset = gidx * 8; \\\n\
        float4 src0 = vload4(0, input_ptr + offset); \\\n\
        float4 src1 = vload4(1, input_ptr + offset); \\\n\
        int4 data0 = convert_int4_rte(src0 * output_scale + output_zp); \\\n\
        int4 data1 = convert_int4_rte(src1 * output_scale + output_zp); \\\n\
        ptr_type dst0, dst1; \\\n\
        _viv_asm(CONV, dst0, data0); \\\n\
        _viv_asm(CONV, dst1, data1); \\\n\
        vstore4(dst0, 0, output_ptr + offset); \\\n\
        vstore4(dst1, 1, output_ptr + offset); \\\n\
    } \\\n\
    for(int i = gidx; i < res; i += get_global_size(0)) \\\n\
    { \\\n\
        float src = input_ptr[length + i]; \\\n\
        int data = convert_int_rte(src * output_scale + output_zp); \\\n\
        output_ptr[length + i] = conv_func(data); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_CONV(U8,  uchar4, 1, uchar, convert_uchar)\n\
SCATTER_ND_UPDATE_CONV(I8,  char4,  1, char,  convert_char)\n\
SCATTER_ND_UPDATE_CONV(I16, short4, 2, short, convert_short)\n\
SCATTER_ND_UPDATE_CONV(F32, float4, 4, float, convert_float)\n\
"; /* end of scatter_nd_update_reduction_conv_cl*/

static const char select_cl[] = "__kernel void select_I8_U8_U8toU8(\n\
    __read_only  image2d_array_t  condition,\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4  value;\n\
    uint4 src0, src1, src, dst;\n\
    float inputScale, inputTail;\n\
    READ_IMAGEI_2DARRAY(value, condition, coord);\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
    src   = (value != 0 ? src0 : src1);\n\
    inputScale = (value.x != 0 ? input0Scale : input1Scale);\n\
    inputTail  = (value.x != 0 ? input0Tail  : input1Tail);\n\
    dst = convert_uint4(convert_float4(src) * inputScale + inputTail);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_U8_U8toU8_2D(\n\
    __read_only  image2d_t        condition,\n\
    __read_only  image2d_t        input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_t        output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4  value = read_imagei(condition, coord);\n\
    uint4 src0  = read_imageui(input0, coord);\n\
    uint4 src1  = read_imageui(input1, coord);\n\
    uint4 src   = (value != 0 ? src0 : src1);\n\
    float inputScale = (value.x != 0 ? input0Scale : input1Scale);\n\
    float inputTail  = (value.x != 0 ? input0Tail  : input1Tail);\n\
    uint4 dst = convert_uint4(convert_float4(src) * inputScale + inputTail);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_I32_I32toI32(\n\
    __read_only  image2d_array_t  condition,\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4  value;\n\
    int4 src0, src1, dst;\n\
    READ_IMAGEI_2DARRAY(value, condition, coord);\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    dst   = (value != 0 ? src0 : src1);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_I32_I32toI32_2D(\n\
    __read_only  image2d_t        condition,\n\
    __read_only  image2d_t        input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_t        output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 value = read_imagei(condition, coord);\n\
    int4 src0  = read_imagei(input0, coord);\n\
    int4 src1  = read_imagei(input1, coord);\n\
    int4 dst   = (value != 0 ? src0 : src1);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_F32_F32toF32(\n\
    __read_only  image2d_array_t  condition,\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4  value;\n\
    float4 src0, src1, dst;\n\
    READ_IMAGEI_2DARRAY(value, condition, coord);\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
    dst   = (value != 0 ? src0 : src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_F32_F32toF32_2D(\n\
    __read_only  image2d_t        condition,\n\
    __read_only  image2d_t        input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_t        output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 value = read_imagei(condition, coord);\n\
    float4 src0  = read_imagef(input0, coord);\n\
    float4 src1  = read_imagef(input1, coord);\n\
    float4 dst   = (value != 0 ? src0 : src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of select_cl*/

static const char sequence_mask_cl[] = "\n\
__kernel void sequence_mask_I32toU8(\n\
    image2d_t input, image2d_array_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    int4 index = read_imagei(input, coord.yz);\n\
    uint4 data;\n\
    data.x = gidx < index.x ? convert_uint_rte(outputVal1) : (uint)(output_ZP);\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toU8_2D(\n\
    image2d_t input, image2d_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int4 index = read_imagei(input, coord.yy);\n\
    uint4 data;\n\
    data.x = gidx < index.x ? convert_uint_rte(outputVal1) : (uint)(output_ZP);\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toI32(\n\
    image2d_t input, image2d_array_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    int4 index = read_imagei(input, coord.yz);\n\
    int4 data;\n\
    data = gidx < index.x ? (int4)(1) : (int4)(0);\n\
    write_imagei(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toI32_2D(\n\
    image2d_t input, image2d_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int4 index = read_imagei(input, coord.yy);\n\
    int4 data;\n\
    data = gidx < index.x ? (int4)(1) : (int4)(0);\n\
    write_imagei(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toF32(\n\
    image2d_t input, image2d_array_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    int4 index = read_imagei(input, coord.yz);\n\
    float4 data;\n\
    data = gidx < index.x ? (float4)(1.0f) : (float4)(0.0f);\n\
    write_imagef(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toF32_2D(\n\
    image2d_t input, image2d_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int4 index = read_imagei(input, coord.yy);\n\
    float4 data;\n\
    data = gidx < index.x ? (float4)(1.0f) : (float4)(0.0f);\n\
    write_imagef(output, coord, data);\n\
}"; /* end of sequence_mask_cl*/

static const char signal_frame_cl[] = "\n\
#define SIGNAL_FRAME_SH_IMPL(type, data_type, read_imagefunc, write_imagefunc) \\\n\
__kernel void signal_frame_##type##to##type \\\n\
    ( \\\n\
    __read_only  image2d_t       input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             frame_step \\\n\
    ) \\\n\
{ \\\n\
    int inner = get_global_id(0); \\\n\
    int length_k = get_global_id(1); \\\n\
    int frames_id = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(inner, length_k, frames_id, frames_id); \\\n\
    int2 coord_in = (int2)(inner, frames_id * frame_step + length_k); \\\n\
 \\\n\
    data_type src = read_imagefunc(input, coord_in); \\\n\
    write_imagefunc(output, coord, src); \\\n\
}\n\
SIGNAL_FRAME_SH_IMPL(F32, float4, read_imagef,  write_imagef)\n\
SIGNAL_FRAME_SH_IMPL(U8,  uint4,  read_imageui, write_imageui)\n\
"; /* end of signal_frame_cl*/

static const char slice_cl[] = "__kernel void slice_F32_I32toF32\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_t       input1,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int4 begin = ((int4 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    float4 src = read_imagef(input0, coord_in);\n\
\n\
    write_imagef(output, coord, src);\n\
}\n\
\n\
__kernel void slice_F32_I32toF32_2D\n\
    (\n\
    __read_only  image2d_t input0,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int2 begin = ((int2 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    float4 src = read_imagef(input0, coord_in);\n\
\n\
    write_imagef(output, coord, src);\n\
}\n\
\n\
__kernel void slice_U8_I32toU8\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_t       input1,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int4 begin = ((int4 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    uint4 src = read_imageui(input0, coord_in);\n\
\n\
    float4 data = convert_float4(src) * inputScale - inputTail;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void slice_U8_I32toU8_2D\n\
    (\n\
    __read_only  image2d_t input0,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int2 begin = ((int2 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    uint4 src = read_imageui(input0, coord_in);\n\
\n\
    float4 data = convert_float4(src) * inputScale - inputTail;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void slice_I32_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_t       input1,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int4 begin = ((int4 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    int4 src = read_imagei(input0, coord_in);\n\
\n\
    write_imagei(output, coord, src);\n\
}\n\
\n\
__kernel void slice_I32_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input0,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int2 begin = ((int2 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    int4 src = read_imagei(input0, coord_in);\n\
\n\
    write_imagei(output, coord, src);\n\
}\n\
\n\
"; /* end of slice_cl*/

static const char space2depth_internal_cl[] = "\n\
__kernel void space2depth_internal_F32toF32 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    float4 data = {0.0};\n\
    data = read_imagef(input, coord);\n\
\n\
    ushort blockSize_x = convert_ushort(block_size_x);\n\
    ushort blockSize_y = convert_ushort(block_size_y);\n\
    int4 coord_out = (int4)(convert_ushort(x)/blockSize_x, convert_ushort(y)/blockSize_y, 0, 0);\n\
    coord_out.z = ((x - coord_out.x * block_size_x) + (y - coord_out.y * block_size_y) * block_size_x) * inDepth\n\
                     + z;\n\
    write_imagef(output, coord_out, data);\n\
}\n\
\n\
__kernel void space2depth_internal_F32toF32_X2Y1 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    float4 data = {0.0};\n\
    data = read_imagef(input, coord);\n\
\n\
    int4 coord_out = (int4)(x >> 1, y, 0, 0);\n\
    coord_out.z = (x & 1) * inDepth + z;\n\
    write_imagef(output, coord_out, data);\n\
}\n\
\n\
__kernel void space2depth_internal_U8toU8 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    uint4 data = {0};\n\
    data = read_imageui(input, coord);\n\
\n\
    ushort blockSize_x = convert_ushort(block_size_x);\n\
    ushort blockSize_y = convert_ushort(block_size_y);\n\
    int4 coord_out = (int4)(convert_ushort(x)/blockSize_x, convert_ushort(y)/blockSize_y, 0, 0);\n\
    coord_out.z = ((x - coord_out.x * block_size_x) + (y - coord_out.y * block_size_y) * block_size_x) * inDepth\n\
                    + z;\n\
\n\
    data.x = convert_uint(data.x * scaleInOut + zpInOut);\n\
    write_imageui(output, coord_out, data);\n\
}\n\
\n\
__kernel void space2depth_internal_U8toU8_X2Y1 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    uint4 data = {0};\n\
    data = read_imageui(input, coord);\n\
\n\
    int4 coord_out = (int4)(x >> 1, y, 0, 0);\n\
    coord_out.z = (x & 1) * inDepth + z;\n\
\n\
    data.x = convert_uint(data.x * scaleInOut + zpInOut);\n\
    write_imageui(output, coord_out, data);\n\
}\n\
"; /* end of space2depth_internal_cl*/

static const char swish_cl[] = "float sigmoid_(float x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
\n\
#define SWISH_F32_F32_PROCESS() \\\n\
    float4 src, tmp, dst; \\\n\
    src   = read_imagef(input, coord); \\\n\
    tmp.x = sigmoid_(src.x * beta, logE); \\\n\
    dst.x = src.x * tmp.x; \\\n\
    write_imagef(output, coord, dst);\n\
\n\
__kernel void swish_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_F32_F32_PROCESS()\n\
}\n\
\n\
__kernel void swish_F32toF32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_F32_F32_PROCESS()\n\
}\n\
\n\
\n\
#define SWISH_U8_U8_PROCESS() \\\n\
    float4 src, tmp, data; \\\n\
    uint4 src0 = read_imageui(input, coord); \\\n\
    src   = convert_float4(src0) * inputScale - inputTail; \\\n\
    tmp.x = sigmoid_(src.x * beta, logE); \\\n\
    data.x = src.x * tmp.x; \\\n\
    uint4 dst = convert_uint4_rte(data * outputScale + outputZP); \\\n\
    write_imageui(output, coord, dst);\n\
\n\
__kernel void swish_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_U8_U8_PROCESS()\n\
}\n\
\n\
__kernel void swish_U8toU8_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_U8_U8_PROCESS()\n\
}\n\
\n\
\n\
#define SWISH_I32_I32_PROCESS() \\\n\
    float4 src, tmp, data; \\\n\
    int4 src0 = read_imagei(input, coord); \\\n\
    src    = convert_float4(src0); \\\n\
    tmp.x  = sigmoid_(src.x * beta, logE); \\\n\
    data.x = src.x * tmp.x; \\\n\
    int4 dst = convert_int4(data); \\\n\
    write_imagei(output, coord, dst);\n\
\n\
__kernel void swish_I32toI32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_I32_I32_PROCESS()\n\
}\n\
\n\
__kernel void swish_I32toI32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_I32_I32_PROCESS()\n\
}\n\
\n\
#define SWISH_F32_U8_PROCESS() \\\n\
    float4 src, tmp, data; \\\n\
    src = read_imagef(input, coord); \\\n\
    tmp.x = sigmoid_(src.x * beta, logE); \\\n\
    data.x = src.x * tmp.x; \\\n\
    uint4 dst = convert_uint4_rte(data * outputScale + outputZP); \\\n\
    write_imageui(output, coord, dst);\n\
\n\
__kernel void swish_F32toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_F32_U8_PROCESS()\n\
}\n\
\n\
__kernel void swish_F32toU8_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_F32_U8_PROCESS()\n\
}"; /* end of swish_cl*/

static const char tile_cl[] = "\n\
#define TILE_3D(name0, name1, src_type, dst_type, conv_type, read_image_func, write_image_func) \\\n\
__kernel void tile_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3, \\\n\
                             float inoutscale, \\\n\
                             float inouttail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out; \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
 \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
 \\\n\
    read_image_func(src, input, coord); \\\n\
 \\\n\
    int batch_id = (short)coord.z / (short)depthIn; \\\n\
    coord.z = (short)coord.z % (short)depthIn; \\\n\
    coord_out = coord; \\\n\
 \\\n\
    for (int w = 0; w < multiples_3; w++) \\\n\
    { \\\n\
        int batch = batchIn * w + batch_id; \\\n\
 \\\n\
        for(int z = 0; z < multiples_2; z++) \\\n\
        { \\\n\
            coord_out.z = coord.z + z * depthIn + batch * depthOut; \\\n\
 \\\n\
            for (int y = 0; y < multiples_1; y++) \\\n\
            { \\\n\
                coord_out.y = coord.y + y * height; \\\n\
 \\\n\
                for (int x = 0; x < multiples_0; x++) \\\n\
                { \\\n\
                    coord_out.x = coord.x + x * width; \\\n\
                    dst = conv_type(convert_float4(src) * inoutscale + inouttail); \\\n\
                    write_image_func(output, coord_out.xyzw, dst); \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
}\n\
TILE_3D(I32, I32, int4,   int4,  convert_int4_rte,  READ_IMAGEI_2DARRAY,  write_imagei)\n\
TILE_3D(U32, U32, uint4,  uint4, convert_uint4_rte, READ_IMAGEUI_2DARRAY, write_imageui)\n\
TILE_3D(F32, F32, float4, float4,convert_float4_rte,READ_IMAGEF_2DARRAY,  write_imagef)\n\
TILE_3D(F32, U32, float4, uint4, convert_uint4_rte, READ_IMAGEF_2DARRAY,  write_imageui)\n\
\n\
#define TILE_2D(name0, name1, src_type, dst_type, conv_type, read_image_func, write_image_func) \\\n\
__kernel void tile_##name0##to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                       int batchIn, \\\n\
                       int depthIn, \\\n\
                       int depthOut, \\\n\
                       int multiples_0, \\\n\
                       int multiples_1, \\\n\
                       int multiples_2, \\\n\
                       int multiples_3, \\\n\
                       float inoutscale, \\\n\
                       float inouttail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
 \\\n\
    src_type src = read_image_func(input, coord); \\\n\
    dst_type dst; \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            dst = conv_type(convert_float4(src) * inoutscale + inouttail); \\\n\
            write_image_func(output, coord, dst); \\\n\
            coord.x += width; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = get_global_id(0); \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D(I32, I32, int4,   int4,  convert_int4_rte,  read_imagei,  write_imagei)\n\
TILE_2D(U32, U32, uint4,  uint4, convert_uint4_rte, read_imageui, write_imageui)\n\
TILE_2D(F32, F32, float4, float4,convert_float4_rte,read_imagef,  write_imagef)\n\
TILE_2D(F32, U32, float4, uint4, convert_uint4_rte, read_imagef,  write_imageui)\n\
\n\
\n\
\n\
"; /* end of tile_cl*/

static const char topk_cl[] = "#define TOPK_F32(LOCAL_SIZE0, STAGES) \\\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE0, 1, 1))) void topk_stage##STAGES##_F32toF32_I32 \\\n\
 ( \\\n\
  __read_only  image2d_t input, \\\n\
  __write_only image2d_t output, \\\n\
  __write_only image2d_t indices, \\\n\
               float     input_scale, \\\n\
               float     input_tail, \\\n\
               float     output_scale, \\\n\
               float     output_tail, \\\n\
               int       num_stages, \\\n\
               int       width \\\n\
  ) \\\n\
 { \\\n\
    uint local_id = get_local_id(0); \\\n\
    uint work_group_size = get_local_size(0); \\\n\
    uint offset = 0; \\\n\
 \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    __local float local_data[LOCAL_SIZE0 * 2]; \\\n\
    __local uint local_indices[LOCAL_SIZE0 * 2]; \\\n\
 \\\n\
    float left = read_imagef(input, coord.xy).x; \\\n\
    coord.z += work_group_size; \\\n\
    float data = read_imagef(input, coord.zy).x; \\\n\
    float right = coord.z < width ? data : -2147483647.0f; \\\n\
 \\\n\
    local_data[local_id] = left; \\\n\
    local_indices[local_id] = local_id; \\\n\
    local_data[local_id + work_group_size] = right; \\\n\
    local_indices[local_id + work_group_size] = local_id + work_group_size; \\\n\
 \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    for (uint stage = 0; stage < num_stages + 1; ++stage) \\\n\
    { \\\n\
        uint signo = (local_id >> stage) & 1; \\\n\
 \\\n\
        for (uint passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
        { \\\n\
            uint postShift = (stage - passOfStage); \\\n\
            uint pairDistance = 1 << postShift; \\\n\
 \\\n\
            uint left_id = ( (local_id >> postShift) << (postShift + 1)) + (local_id & (pairDistance - 1)); \\\n\
            uint right_id = left_id + pairDistance; \\\n\
 \\\n\
            uint left_idx = local_indices[left_id]; \\\n\
            uint right_idx = local_indices[right_id]; \\\n\
 \\\n\
            float left_elem = local_data[left_id]; \\\n\
            float right_elem = local_data[right_id]; \\\n\
 \\\n\
            if ((left_elem < right_elem || (left_elem == right_elem && left_idx < right_idx)) ^ signo) \\\n\
            { \\\n\
                local_data[left_id] = right_elem; \\\n\
                local_data[right_id] = left_elem; \\\n\
 \\\n\
                local_indices[left_id] = right_idx; \\\n\
                local_indices[right_id] = left_idx; \\\n\
            } \\\n\
 \\\n\
            barrier(CLK_LOCAL_MEM_FENCE); \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    float4 dst; \\\n\
    dst.x = local_data[local_id]; \\\n\
    dst.y = local_data[local_id + work_group_size]; \\\n\
 \\\n\
    write_imagef(output, coord.xy, dst.xxxx); \\\n\
    write_imagef(output, coord.zy, dst.yyyy); \\\n\
 \\\n\
    int4 index; \\\n\
    index.x = ((int*)local_indices)[local_id]; \\\n\
    index.y = ((int*)local_indices)[local_id + work_group_size]; \\\n\
 \\\n\
    write_imagei(indices, coord.xy, index.xxxx); \\\n\
    write_imagei(indices, coord.zy, index.yyyy); \\\n\
 }\n\
TOPK_F32((1 << 0), 0)\n\
TOPK_F32((1 << 1), 1)\n\
TOPK_F32((1 << 2), 2)\n\
TOPK_F32((1 << 3), 3)\n\
TOPK_F32((1 << 4), 4)\n\
TOPK_F32((1 << 5), 5)\n\
TOPK_F32((1 << 6), 6)\n\
TOPK_F32((1 << 9), 9)\n\
\n\
#define TOPK_U32(LOCAL_SIZE0, STAGES) \\\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE0, 1, 1))) void topk_stage##STAGES##_U32toU32_I32 \\\n\
 ( \\\n\
  __read_only  image2d_t input, \\\n\
  __write_only image2d_t output, \\\n\
  __write_only image2d_t indices, \\\n\
               float     input_scale, \\\n\
               float     input_tail, \\\n\
               float     output_scale, \\\n\
               float     output_tail, \\\n\
               int       num_stages, \\\n\
               int       width \\\n\
  ) \\\n\
 { \\\n\
    uint local_id = get_local_id(0); \\\n\
    uint work_group_size = get_local_size(0); \\\n\
    uint offset = 0; \\\n\
 \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    __local uint local_data[LOCAL_SIZE0 * 2]; \\\n\
    __local uint local_indices[LOCAL_SIZE0 * 2]; \\\n\
 \\\n\
    uint left = read_imageui(input, coord.xy).x; \\\n\
    coord.z += work_group_size; \\\n\
    uint data = read_imageui(input, coord.zy).x; \\\n\
    uint right = coord.z < width ? data : 0; \\\n\
 \\\n\
    local_data[local_id] = left; \\\n\
    local_indices[local_id] = local_id; \\\n\
    local_data[local_id + work_group_size] = right; \\\n\
    local_indices[local_id + work_group_size] = local_id + work_group_size; \\\n\
 \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    for (uint stage = 0; stage < num_stages + 1; ++stage) \\\n\
    { \\\n\
        uint signo = (local_id >> stage) & 1; \\\n\
 \\\n\
        for (uint passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
        { \\\n\
            uint postShift = (stage - passOfStage); \\\n\
            uint pairDistance = 1 << postShift; \\\n\
 \\\n\
            uint left_id = ( (local_id >> postShift) << (postShift + 1)) + (local_id & (pairDistance - 1)); \\\n\
            uint right_id = left_id + pairDistance; \\\n\
 \\\n\
            uint left_idx = local_indices[left_id]; \\\n\
            uint right_idx = local_indices[right_id]; \\\n\
 \\\n\
            uint left_elem = local_data[left_id]; \\\n\
            uint right_elem = local_data[right_id]; \\\n\
 \\\n\
            if ((left_elem < right_elem || (left_elem == right_elem && left_idx < right_idx)) ^ signo) \\\n\
            { \\\n\
                local_data[left_id] = right_elem; \\\n\
                local_data[right_id] = left_elem; \\\n\
 \\\n\
                local_indices[left_id] = right_idx; \\\n\
                local_indices[right_id] = left_idx; \\\n\
            } \\\n\
 \\\n\
            barrier(CLK_LOCAL_MEM_FENCE); \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    uint4 dst; \\\n\
    dst.x = local_data[local_id]; \\\n\
    dst.y = local_data[local_id + work_group_size]; \\\n\
 \\\n\
    write_imageui(output, coord.xy, dst.xxxx); \\\n\
    write_imageui(output, coord.zy, dst.yyyy); \\\n\
 \\\n\
    int4 index; \\\n\
    index.x = ((int*)local_indices)[local_id]; \\\n\
    index.y = ((int*)local_indices)[local_id + work_group_size]; \\\n\
 \\\n\
    write_imagei(indices, coord.xy, index.xxxx); \\\n\
    write_imagei(indices, coord.zy, index.yyyy); \\\n\
 }\n\
TOPK_U32((1 << 0), 0)\n\
TOPK_U32((1 << 1), 1)\n\
TOPK_U32((1 << 2), 2)\n\
TOPK_U32((1 << 3), 3)\n\
TOPK_U32((1 << 4), 4)\n\
TOPK_U32((1 << 5), 5)\n\
TOPK_U32((1 << 6), 6)\n\
TOPK_U32((1 << 9), 9)\n\
\n\
#define TOPK_I32(LOCAL_SIZE0, STAGES) \\\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE0, 1, 1))) void topk_stage##STAGES##_I32toI32_I32 \\\n\
 ( \\\n\
  __read_only  image2d_t input, \\\n\
  __write_only image2d_t output, \\\n\
  __write_only image2d_t indices, \\\n\
               float     input_scale, \\\n\
               float     input_tail, \\\n\
               float     output_scale, \\\n\
               float     output_tail, \\\n\
               int       num_stages, \\\n\
               int       width \\\n\
  ) \\\n\
 { \\\n\
    int local_id = get_local_id(0); \\\n\
    int work_group_size = get_local_size(0); \\\n\
    int offset = 0; \\\n\
 \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    __local int local_data[LOCAL_SIZE0 * 2]; \\\n\
    __local int local_indices[LOCAL_SIZE0 * 2]; \\\n\
 \\\n\
    int left = read_imagei(input, coord.xy).x; \\\n\
    coord.z += work_group_size; \\\n\
    int data = read_imagei(input, coord.zy).x; \\\n\
    int right = coord.z < width ? data : -2147483647; \\\n\
 \\\n\
    local_data[local_id] = left; \\\n\
    local_indices[local_id] = local_id; \\\n\
    local_data[local_id + work_group_size] = right; \\\n\
    local_indices[local_id + work_group_size] = local_id + work_group_size; \\\n\
 \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    for (int stage = 0; stage < num_stages + 1; ++stage) \\\n\
    { \\\n\
        int signo = (local_id >> stage) & 1; \\\n\
 \\\n\
        for (int passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
        { \\\n\
            int postShift = (stage - passOfStage); \\\n\
            int pairDistance = 1 << postShift; \\\n\
 \\\n\
            int left_id = ( (local_id >> postShift) << (postShift + 1)) + (local_id & (pairDistance - 1)); \\\n\
            int right_id = left_id + pairDistance; \\\n\
 \\\n\
            int left_idx = local_indices[left_id]; \\\n\
            int right_idx = local_indices[right_id]; \\\n\
 \\\n\
            int left_elem = local_data[left_id]; \\\n\
            int right_elem = local_data[right_id]; \\\n\
 \\\n\
            if ((left_elem < right_elem || (left_elem == right_elem && left_idx < right_idx)) ^ signo) \\\n\
            { \\\n\
                local_data[left_id] = right_elem; \\\n\
                local_data[right_id] = left_elem; \\\n\
 \\\n\
                local_indices[left_id] = right_idx; \\\n\
                local_indices[right_id] = left_idx; \\\n\
            } \\\n\
 \\\n\
            barrier(CLK_LOCAL_MEM_FENCE); \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    int4 dst; \\\n\
    dst.x = local_data[local_id]; \\\n\
    dst.y = local_data[local_id + work_group_size]; \\\n\
 \\\n\
    write_imagei(output, coord.xy, dst.xxxx); \\\n\
    write_imagei(output, coord.zy, dst.yyyy); \\\n\
 \\\n\
    int4 index; \\\n\
    index.x = ((int*)local_indices)[local_id]; \\\n\
    index.y = ((int*)local_indices)[local_id + work_group_size]; \\\n\
 \\\n\
    write_imagei(indices, coord.xy, index.xxxx); \\\n\
    write_imagei(indices, coord.zy, index.yyyy); \\\n\
 }\n\
TOPK_I32((1 << 0), 0)\n\
TOPK_I32((1 << 1), 1)\n\
TOPK_I32((1 << 2), 2)\n\
TOPK_I32((1 << 3), 3)\n\
TOPK_I32((1 << 4), 4)\n\
TOPK_I32((1 << 5), 5)\n\
TOPK_I32((1 << 6), 6)\n\
TOPK_I32((1 << 9), 9)\n\
\n\
#define TOPK_F32toU32(LOCAL_SIZE0, STAGES) \\\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE0, 1, 1))) void topk_stage##STAGES##_F32toU32_I32 \\\n\
 ( \\\n\
  __read_only  image2d_t input, \\\n\
  __write_only image2d_t output, \\\n\
  __write_only image2d_t indices, \\\n\
               float     input_scale, \\\n\
               float     input_tail, \\\n\
               float     output_scale, \\\n\
               float     output_tail, \\\n\
               int       num_stages, \\\n\
               int       width \\\n\
  ) \\\n\
 { \\\n\
    uint local_id = get_local_id(0); \\\n\
    uint work_group_size = get_local_size(0); \\\n\
    uint offset = 0; \\\n\
 \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    __local float local_data[LOCAL_SIZE0 * 2]; \\\n\
    __local uint local_indices[LOCAL_SIZE0 * 2]; \\\n\
 \\\n\
    float left = read_imagef(input, coord.xy).x; \\\n\
    coord.z += work_group_size; \\\n\
    float data = read_imagef(input, coord.zy).x; \\\n\
    float right = coord.z < width ? data : -2147483647.0f; \\\n\
 \\\n\
    local_data[local_id] = left; \\\n\
    local_indices[local_id] = local_id; \\\n\
    local_data[local_id + work_group_size] = right; \\\n\
    local_indices[local_id + work_group_size] = local_id + work_group_size; \\\n\
 \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    for (uint stage = 0; stage < num_stages + 1; ++stage) \\\n\
    { \\\n\
        uint signo = (local_id >> stage) & 1; \\\n\
 \\\n\
        for (uint passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
        { \\\n\
            uint postShift = (stage - passOfStage); \\\n\
            uint pairDistance = 1 << postShift; \\\n\
 \\\n\
            uint left_id = ( (local_id >> postShift) << (postShift + 1)) + (local_id & (pairDistance - 1)); \\\n\
            uint right_id = left_id + pairDistance; \\\n\
 \\\n\
            uint left_idx = local_indices[left_id]; \\\n\
            uint right_idx = local_indices[right_id]; \\\n\
 \\\n\
            float left_elem = local_data[left_id]; \\\n\
            float right_elem = local_data[right_id]; \\\n\
 \\\n\
            if ((left_elem < right_elem || (left_elem == right_elem && left_idx < right_idx)) ^ signo) \\\n\
            { \\\n\
                local_data[left_id] = right_elem; \\\n\
                local_data[right_id] = left_elem; \\\n\
 \\\n\
                local_indices[left_id] = right_idx; \\\n\
                local_indices[right_id] = left_idx; \\\n\
            } \\\n\
 \\\n\
            barrier(CLK_LOCAL_MEM_FENCE); \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    uint4 dst; \\\n\
    dst.x = convert_uint(local_data[local_id] * output_scale + output_tail); \\\n\
    dst.y = convert_uint(local_data[local_id + work_group_size] * output_scale + output_tail); \\\n\
    write_imageui(output, coord.xy, dst.xxxx); \\\n\
    write_imageui(output, coord.zy, dst.yyyy); \\\n\
 \\\n\
    int4 index; \\\n\
    index.x = ((int*)local_indices)[local_id]; \\\n\
    index.y = ((int*)local_indices)[local_id + work_group_size]; \\\n\
 \\\n\
    write_imagei(indices, coord.xy, index.xxxx); \\\n\
    write_imagei(indices, coord.zy, index.yyyy); \\\n\
 }\n\
\n\
TOPK_F32toU32((1 << 0), 0)\n\
TOPK_F32toU32((1 << 1), 1)\n\
TOPK_F32toU32((1 << 2), 2)\n\
TOPK_F32toU32((1 << 3), 3)\n\
TOPK_F32toU32((1 << 4), 4)\n\
TOPK_F32toU32((1 << 5), 5)\n\
TOPK_F32toU32((1 << 6), 6)\n\
TOPK_F32toU32((1 << 9), 9)\n\
\n\
#define TOPK_F32toI32(LOCAL_SIZE0, STAGES) \\\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE0, 1, 1))) void topk_stage##STAGES##_F32toI32_I32 \\\n\
 ( \\\n\
  __read_only  image2d_t input, \\\n\
  __write_only image2d_t output, \\\n\
  __write_only image2d_t indices, \\\n\
               float     input_scale, \\\n\
               float     input_tail, \\\n\
               float     output_scale, \\\n\
               float     output_tail, \\\n\
               int       num_stages, \\\n\
               int       width \\\n\
  ) \\\n\
 { \\\n\
    uint local_id = get_local_id(0); \\\n\
    uint work_group_size = get_local_size(0); \\\n\
    uint offset = 0; \\\n\
 \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    __local float local_data[LOCAL_SIZE0 * 2]; \\\n\
    __local uint local_indices[LOCAL_SIZE0 * 2]; \\\n\
 \\\n\
    float left = read_imagef(input, coord.xy).x; \\\n\
    coord.z += work_group_size; \\\n\
    float data = read_imagef(input, coord.zy).x; \\\n\
    float right = coord.z < width ? data : -2147483647.0f; \\\n\
 \\\n\
    local_data[local_id] = left; \\\n\
    local_indices[local_id] = local_id; \\\n\
    local_data[local_id + work_group_size] = right; \\\n\
    local_indices[local_id + work_group_size] = local_id + work_group_size; \\\n\
 \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    for (uint stage = 0; stage < num_stages + 1; ++stage) \\\n\
    { \\\n\
        uint signo = (local_id >> stage) & 1; \\\n\
 \\\n\
        for (uint passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
        { \\\n\
            uint postShift = (stage - passOfStage); \\\n\
            uint pairDistance = 1 << postShift; \\\n\
 \\\n\
            uint left_id = ( (local_id >> postShift) << (postShift + 1)) + (local_id & (pairDistance - 1)); \\\n\
            uint right_id = left_id + pairDistance; \\\n\
 \\\n\
            uint left_idx = local_indices[left_id]; \\\n\
            uint right_idx = local_indices[right_id]; \\\n\
 \\\n\
            float left_elem = local_data[left_id]; \\\n\
            float right_elem = local_data[right_id]; \\\n\
 \\\n\
            if ((left_elem < right_elem || (left_elem == right_elem && left_idx < right_idx)) ^ signo) \\\n\
            { \\\n\
                local_data[left_id] = right_elem; \\\n\
                local_data[right_id] = left_elem; \\\n\
 \\\n\
                local_indices[left_id] = right_idx; \\\n\
                local_indices[right_id] = left_idx; \\\n\
            } \\\n\
 \\\n\
            barrier(CLK_LOCAL_MEM_FENCE); \\\n\
        } \\\n\
    } \\\n\
 \\\n\
    int4 dst; \\\n\
    dst.x = convert_int(local_data[local_id] * output_scale + output_tail); \\\n\
    dst.y = convert_int(local_data[local_id + work_group_size] * output_scale + output_tail); \\\n\
    write_imagei(output, coord.xy, dst.xxxx); \\\n\
    write_imagei(output, coord.zy, dst.yyyy); \\\n\
 \\\n\
    int4 index; \\\n\
    index.x = ((int*)local_indices)[local_id]; \\\n\
    index.y = ((int*)local_indices)[local_id + work_group_size]; \\\n\
 \\\n\
    write_imagei(indices, coord.xy, index.xxxx); \\\n\
    write_imagei(indices, coord.zy, index.yyyy); \\\n\
 }\n\
\n\
TOPK_F32toI32((1 << 0), 0)\n\
TOPK_F32toI32((1 << 1), 1)\n\
TOPK_F32toI32((1 << 2), 2)\n\
TOPK_F32toI32((1 << 3), 3)\n\
TOPK_F32toI32((1 << 4), 4)\n\
TOPK_F32toI32((1 << 5), 5)\n\
TOPK_F32toI32((1 << 6), 6)\n\
TOPK_F32toI32((1 << 9), 9)"; /* end of topk_cl*/

static const char topk2_cl[] = "\n\
#define BITONIC_STEP(dtype) \\\n\
void bitonic_step_##dtype(uint num_stages, int lx, \\\n\
        __local dtype *local_data, __local int *local_indices) \\\n\
{ \\\n\
    for (uint stage = 0; stage < num_stages + 1; ++stage) \\\n\
    { \\\n\
        uint signo = (lx >> stage) & 1; \\\n\
 \\\n\
        for (uint passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
        { \\\n\
            uint postShift = (stage - passOfStage); \\\n\
            uint pairDistance = 1 << postShift; \\\n\
 \\\n\
            uint left_id = ( (lx >> postShift) << (postShift + 1)) + (lx & (pairDistance - 1)); \\\n\
            uint right_id = left_id + pairDistance; \\\n\
 \\\n\
            int left_idx = local_indices[left_id]; \\\n\
            int right_idx = local_indices[right_id]; \\\n\
 \\\n\
            dtype left_elem = local_data[left_id]; \\\n\
            dtype right_elem = local_data[right_id]; \\\n\
 \\\n\
            if ((left_elem < right_elem || (left_elem == right_elem && left_idx < right_idx)) ^ signo) \\\n\
            { \\\n\
                local_data[left_id] = right_elem; \\\n\
                local_data[right_id] = left_elem; \\\n\
 \\\n\
                local_indices[left_id] = right_idx; \\\n\
                local_indices[right_id] = left_idx; \\\n\
            } \\\n\
 \\\n\
            barrier(CLK_LOCAL_MEM_FENCE); \\\n\
        } \\\n\
    } \\\n\
}\n\
BITONIC_STEP(int)\n\
BITONIC_STEP(uint)\n\
\n\
#define BITONIC_STEP_ASCEND(dtype) \\\n\
void bitonic_step_ascend_##dtype(uint num_stages, int lx, \\\n\
        __local dtype *p_share_k, __local int *p_share_v) \\\n\
{ \\\n\
    for (uint stage = 0; stage < num_stages + 1; ++stage) \\\n\
    { \\\n\
        uint signo = (lx >> stage) & 1; \\\n\
 \\\n\
        for (uint passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
        { \\\n\
            uint postShift = (stage - passOfStage); \\\n\
            uint pairDistance = 1 << postShift; \\\n\
 \\\n\
            uint left_id = ( (lx >> postShift) << (postShift + 1)) + (lx & (pairDistance - 1)); \\\n\
            uint right_id = left_id + pairDistance; \\\n\
 \\\n\
            int left_idx = p_share_v[left_id]; \\\n\
            int right_idx = p_share_v[right_id]; \\\n\
 \\\n\
            dtype left_elem = p_share_k[left_id]; \\\n\
            dtype right_elem = p_share_k[right_id]; \\\n\
 \\\n\
            if ((left_elem > right_elem || (left_elem == right_elem && left_idx > right_idx)) ^ signo) \\\n\
            { \\\n\
                p_share_k[left_id] = right_elem; \\\n\
                p_share_k[right_id] = left_elem; \\\n\
 \\\n\
                p_share_v[left_id] = right_idx; \\\n\
                p_share_v[right_id] = left_idx; \\\n\
            } \\\n\
 \\\n\
            barrier(CLK_LOCAL_MEM_FENCE); \\\n\
        } \\\n\
    } \\\n\
}\n\
BITONIC_STEP_ASCEND(int)\n\
BITONIC_STEP_ASCEND(uint)\n\
\n\
#define BITONIC_MERGE(dtype) \\\n\
void bitonic_merge_##dtype(uint num_stages, int lx, \\\n\
        __local dtype *local_data, __local int *local_indices) \\\n\
{ \\\n\
    uint stage = num_stages; \\\n\
    uint signo = (lx >> stage) & 1; \\\n\
 \\\n\
    for (uint passOfStage = 0; passOfStage < stage + 1; ++passOfStage) \\\n\
    { \\\n\
        uint postShift = (stage - passOfStage); \\\n\
        uint pairDistance = 1 << postShift; \\\n\
 \\\n\
        uint left_id = ( (lx >> postShift) << (postShift + 1)) + (lx & (pairDistance - 1)); \\\n\
        uint right_id = left_id + pairDistance; \\\n\
 \\\n\
        int left_idx = local_indices[left_id]; \\\n\
        int right_idx = local_indices[right_id]; \\\n\
 \\\n\
        dtype left_elem = local_data[left_id]; \\\n\
        dtype right_elem = local_data[right_id]; \\\n\
 \\\n\
        if ((left_elem < right_elem || (left_elem == right_elem && left_idx < right_idx)) ^ signo) \\\n\
        { \\\n\
            local_data[left_id] = right_elem; \\\n\
            local_data[right_id] = left_elem; \\\n\
 \\\n\
            local_indices[left_id] = right_idx; \\\n\
            local_indices[right_id] = left_idx; \\\n\
        } \\\n\
 \\\n\
        barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    } \\\n\
}\n\
BITONIC_MERGE(int)\n\
BITONIC_MERGE(uint)\n\
\n\
#define BLOCK_SIZE              (512)\n\
\n\
__kernel __attribute__((reqd_work_group_size(BLOCK_SIZE, 1, 1))) void topk_stage_I32toI32_I32\n\
(\n\
  __read_only  image2d_t input,\n\
  __write_only image2d_t output,\n\
  __write_only image2d_t indices,\n\
               float     input_scale,\n\
               float     input_tail,\n\
               float     output_scale,\n\
               float     output_tail,\n\
               int       _num_stages,\n\
               int       width\n\
  )\n\
 {\n\
    uint lx = get_local_id(0);\n\
    const int init_k = -2147483647;\n\
    const int init_v = -2147483647;\n\
    const int num_stages = 9;\n\
    const int threads_per_block = BLOCK_SIZE;\n\
    const int index_minus_1 = threads_per_block * 2 - 1;\n\
    uint offset = 0;\n\
    uint lx1 = lx + threads_per_block;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    __local int local_data[1536];\n\
    __local int local_indices[1536];\n\
\n\
    int left = read_imagei(input, coord.xy).x;\n\
    coord.z += threads_per_block;\n\
    int right = read_imagei(input, coord.zy).x;\n\
\n\
    local_data[lx] = left;\n\
    local_indices[lx] = coord.x;\n\
    local_data[lx1] = right;\n\
    local_indices[lx1] = coord.z;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    bitonic_step_int(num_stages, lx, local_data, local_indices);\n\
\n\
    int min_data = local_data[511];\n\
\n\
    int *p_share_k = local_data + threads_per_block;\n\
    int *p_share_v = local_indices + threads_per_block;\n\
\n\
    int limit = (width >> 10) << 10;\n\
    p_share_k[lx] = init_k;\n\
    p_share_v[lx] = init_v;\n\
\n\
    p_share_k[lx1] = init_k;\n\
    p_share_v[lx1] = init_v;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    for (coord.x = lx + threads_per_block * 2; coord.x < limit; coord.x = coord.x + threads_per_block * 2)\n\
    {\n\
        int2 data;\n\
        coord.z = coord.x + threads_per_block;\n\
        data.x = read_imagei(input, coord.xy).x;\n\
        data.y = read_imagei(input, coord.zy).x;\n\
\n\
        p_share_k[lx] = data.x;\n\
        p_share_v[lx] = coord.x;\n\
\n\
        p_share_k[lx1] = data.y;\n\
        p_share_v[lx1] = coord.z;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        bitonic_step_ascend_int(num_stages, lx, p_share_k, p_share_v);\n\
\n\
        if (p_share_k[index_minus_1] < min_data)\n\
        {\n\
            continue;\n\
        }\n\
\n\
        p_share_k[lx] = p_share_k[lx1];\n\
        p_share_v[lx] = p_share_v[lx1];\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        bitonic_merge_int(num_stages, lx, local_data, local_indices);\n\
\n\
        min_data = local_data[511];\n\
        p_share_k[lx] = init_k;\n\
        p_share_v[lx] = init_v;\n\
        p_share_k[lx1] = init_k;\n\
        p_share_v[lx1] = init_v;\n\
    }\n\
\n\
    if (width > limit)\n\
    {\n\
        if (coord.x < width)\n\
        {\n\
            int2 data;\n\
            data.x = read_imagei(input, coord.xy).x;\n\
            coord.z = coord.x + threads_per_block;\n\
            data.y = read_imagei(input, coord.zy).x;\n\
\n\
            p_share_k[lx] = data.x;\n\
            p_share_v[lx] = coord.x;\n\
\n\
            p_share_k[lx1] = coord.z < width ? data.y : init_k;\n\
            p_share_v[lx1] = coord.z < width ? coord.z : init_v;\n\
        }\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        bitonic_step_ascend_int(num_stages, lx, p_share_k, p_share_v);\n\
\n\
        if (p_share_k[index_minus_1] >= min_data)\n\
        {\n\
            p_share_k[lx] = p_share_k[lx1];\n\
            p_share_v[lx] = p_share_v[lx1];\n\
            barrier(CLK_LOCAL_MEM_FENCE);\n\
            bitonic_merge_int(num_stages, lx, local_data, local_indices);\n\
        }\n\
    }\n\
\n\
    int4 dst;\n\
    dst.x = local_data[lx];\n\
\n\
    coord.x = lx;\n\
    write_imagei(output, coord.xy, dst.xxxx);\n\
\n\
    int4 index;\n\
    index.x = local_indices[lx];\n\
\n\
    write_imagei(indices, coord.xy, index.xxxx);\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(BLOCK_SIZE, 1, 1))) void topk_stage_U32toU32_I32\n\
(\n\
  __read_only  image2d_t input,\n\
  __write_only image2d_t output,\n\
  __write_only image2d_t indices,\n\
               float     input_scale,\n\
               float     input_tail,\n\
               float     output_scale,\n\
               float     output_tail,\n\
               int       _num_stages,\n\
               int       width\n\
  )\n\
 {\n\
    uint lx = get_local_id(0);\n\
    const uint init_k = 0;\n\
    const int init_v = -2147483647;\n\
    const int num_stages = 9;\n\
    const int threads_per_block = BLOCK_SIZE;\n\
    const int index_minus_1 = threads_per_block * 2 - 1;\n\
    uint offset = 0;\n\
    uint lx1 = lx + threads_per_block;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    __local uint local_data[1536];\n\
    __local int local_indices[1536];\n\
\n\
    uint left = read_imageui(input, coord.xy).x;\n\
    coord.z += threads_per_block;\n\
    uint right = read_imageui(input, coord.zy).x;\n\
\n\
    local_data[lx] = left;\n\
    local_indices[lx] = coord.x;\n\
    local_data[lx1] = right;\n\
    local_indices[lx1] = coord.z;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    bitonic_step_uint(num_stages, lx, local_data, local_indices);\n\
\n\
    uint min_data = local_data[511];\n\
\n\
    uint *p_share_k = local_data + threads_per_block;\n\
    int *p_share_v = local_indices + threads_per_block;\n\
\n\
    int limit = (width >> 10) << 10;\n\
    p_share_k[lx] = init_k;\n\
    p_share_v[lx] = init_v;\n\
\n\
    p_share_k[lx1] = init_k;\n\
    p_share_v[lx1] = init_v;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    for (coord.x = lx + threads_per_block * 2; coord.x < limit; coord.x = coord.x + threads_per_block * 2)\n\
    {\n\
        uint2 data;\n\
        coord.z = coord.x + threads_per_block;\n\
        data.x = read_imageui(input, coord.xy).x;\n\
        data.y = read_imageui(input, coord.zy).x;\n\
\n\
        p_share_k[lx] = data.x;\n\
        p_share_v[lx] = coord.x;\n\
\n\
        p_share_k[lx1] = data.y;\n\
        p_share_v[lx1] = coord.z;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        bitonic_step_ascend_uint(num_stages, lx, p_share_k, p_share_v);\n\
\n\
        if (p_share_k[index_minus_1] < min_data)\n\
        {\n\
            continue;\n\
        }\n\
\n\
        p_share_k[lx] = p_share_k[lx1];\n\
        p_share_v[lx] = p_share_v[lx1];\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        bitonic_merge_uint(num_stages, lx, local_data, local_indices);\n\
\n\
        min_data = local_data[511];\n\
        p_share_k[lx] = init_k;\n\
        p_share_v[lx] = init_v;\n\
        p_share_k[lx1] = init_k;\n\
        p_share_v[lx1] = init_v;\n\
    }\n\
\n\
    if (width > limit)\n\
    {\n\
        if (coord.x < width)\n\
        {\n\
            uint2 data;\n\
            data.x = read_imageui(input, coord.xy).x;\n\
            coord.z = coord.x + threads_per_block;\n\
            data.y = read_imageui(input, coord.zy).x;\n\
\n\
            p_share_k[lx] = data.x;\n\
            p_share_v[lx] = coord.x;\n\
\n\
            p_share_k[lx1] = coord.z < width ? data.y : init_k;\n\
            p_share_v[lx1] = coord.z < width ? coord.z : init_v;\n\
        }\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        bitonic_step_ascend_uint(num_stages, lx, p_share_k, p_share_v);\n\
\n\
        if (p_share_k[index_minus_1] >= min_data)\n\
        {\n\
            p_share_k[lx] = p_share_k[lx1];\n\
            p_share_v[lx] = p_share_v[lx1];\n\
            barrier(CLK_LOCAL_MEM_FENCE);\n\
            bitonic_merge_uint(num_stages, lx, local_data, local_indices);\n\
        }\n\
    }\n\
\n\
    uint4 dst;\n\
    dst.x = local_data[lx];\n\
\n\
    coord.x = lx;\n\
    write_imageui(output, coord.xy, dst.xxxx);\n\
\n\
    int4 index;\n\
    index.x = local_indices[lx];\n\
\n\
    write_imagei(indices, coord.xy, index.xxxx);\n\
}\n\
"; /* end of topk2_cl*/

static const char topk_odd_even_sort_cl[] = "#define LOCAL_SIZE_X    (32)\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X, 1, 1))) void topk_odd_even_sort_F32toF32_I32\n\
 (\n\
  __read_only  image2d_t input,\n\
               image2d_t input_t,\n\
               image2d_t indices_t,\n\
  __write_only image2d_t output,\n\
  __write_only image2d_t indices,\n\
               float     input_scale,\n\
               float     input_tail,\n\
               float     output_scale,\n\
               float     output_tail,\n\
               int       width\n\
  )\n\
 {\n\
    uint lid = get_local_id(0);\n\
    uint work_group_size = get_local_size(0);\n\
    uint offset = 0;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        float4 data = read_imagef(input, coord.xy);\n\
\n\
        write_imagef(input_t, coord.xy, data);\n\
        write_imagei(indices_t, coord.xy, coord.xxxx);\n\
    }\n\
\n\
    __local int sorted[1];\n\
\n\
    sorted[0] = 0;\n\
\n\
    while (1)\n\
    {\n\
        if (lid == 0)\n\
        {\n\
            *sorted = 0;\n\
        }\n\
        int swapped = 0;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        // odd-even\n\
        coord.x = lid * 2;\n\
        coord.z = lid * 2 + 1;\n\
        for (; coord.z < width; )\n\
        {\n\
            float4 left = read_imagef(input_t, coord.xy);\n\
            float4 right = read_imagef(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imagef(input_t, coord.xy, right);\n\
                write_imagef(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
        // even-odd\n\
        coord.x = lid * 2 + 1;\n\
        coord.z = lid * 2 + 2;\n\
        for (; coord.z < width; )\n\
        {\n\
            float4 left = read_imagef(input_t, coord.xy);\n\
            float4 right = read_imagef(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imagef(input_t, coord.xy, right);\n\
                write_imagef(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        atomic_add(sorted, swapped);\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        if (*sorted == 0)\n\
            break;\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
    }\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        float4 data = read_imagef(input_t, coord.xy);\n\
        int4 index = read_imagei(indices_t, coord.xy);\n\
\n\
        write_imagef(output, coord.xy, data);\n\
        write_imagei(indices, coord.xy, index);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X, 1, 1))) void topk_odd_even_sort_U32toU32_I32\n\
 (\n\
  __read_only  image2d_t input,\n\
               image2d_t input_t,\n\
               image2d_t indices_t,\n\
  __write_only image2d_t output,\n\
  __write_only image2d_t indices,\n\
               float     input_scale,\n\
               float     input_tail,\n\
               float     output_scale,\n\
               float     output_tail,\n\
               int       width\n\
  )\n\
 {\n\
    uint lid = get_local_id(0);\n\
    uint work_group_size = get_local_size(0);\n\
    uint offset = 0;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        uint4 data = read_imageui(input, coord.xy);\n\
\n\
        write_imageui(input_t, coord.xy, data);\n\
        write_imagei(indices_t, coord.xy, coord.xxxx);\n\
    }\n\
\n\
    __local int sorted[1];\n\
    sorted[0] = 0;\n\
\n\
    while (1)\n\
    {\n\
        if (lid == 0)\n\
        {\n\
            *sorted = 0;\n\
        }\n\
        int swapped = 0;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        // odd-even\n\
        coord.x = lid * 2;\n\
        coord.z = lid * 2 + 1;\n\
        for (; coord.z < width; )\n\
        {\n\
            uint4 left = read_imageui(input_t, coord.xy);\n\
            uint4 right = read_imageui(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imageui(input_t, coord.xy, right);\n\
                write_imageui(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
        // even-odd\n\
        coord.x = lid * 2 + 1;\n\
        coord.z = lid * 2 + 2;\n\
        for (; coord.z < width; )\n\
        {\n\
            uint4 left = read_imageui(input_t, coord.xy);\n\
            uint4 right = read_imageui(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imageui(input_t, coord.xy, right);\n\
                write_imageui(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        atomic_add(sorted, swapped);\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        if (*sorted == 0)\n\
            break;\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
    }\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        uint4 data = read_imageui(input_t, coord.xy);\n\
        int4 index = read_imagei(indices_t, coord.xy);\n\
\n\
        write_imageui(output, coord.xy, data);\n\
        write_imagei(indices, coord.xy, index);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X, 1, 1))) void topk_odd_even_sort_I32toI32_I32\n\
 (\n\
  __read_only  image2d_t input,\n\
               image2d_t input_t,\n\
               image2d_t indices_t,\n\
  __write_only image2d_t output,\n\
  __write_only image2d_t indices,\n\
               float     input_scale,\n\
               float     input_tail,\n\
               float     output_scale,\n\
               float     output_tail,\n\
               int       width\n\
  )\n\
 {\n\
    uint lid = get_local_id(0);\n\
    uint work_group_size = get_local_size(0);\n\
    uint offset = 0;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        int4 data = read_imagei(input, coord.xy);\n\
\n\
        write_imagei(input_t, coord.xy, data);\n\
        write_imagei(indices_t, coord.xy, coord.xxxx);\n\
    }\n\
\n\
    __local int sorted[1];\n\
    sorted[0] = 0;\n\
\n\
    while (1)\n\
    {\n\
        if (lid == 0)\n\
        {\n\
            *sorted = 0;\n\
        }\n\
        int swapped = 0;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        // odd-even\n\
        coord.x = lid * 2;\n\
        coord.z = lid * 2 + 1;\n\
        for (; coord.z < width; )\n\
        {\n\
            int4 left = read_imagei(input_t, coord.xy);\n\
            int4 right = read_imagei(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imagei(input_t, coord.xy, right);\n\
                write_imagei(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
        // even-odd\n\
        coord.x = lid * 2 + 1;\n\
        coord.z = lid * 2 + 2;\n\
        for (; coord.z < width; )\n\
        {\n\
            int4 left = read_imagei(input_t, coord.xy);\n\
            int4 right = read_imagei(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imagei(input_t, coord.xy, right);\n\
                write_imagei(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        atomic_add(sorted, swapped);\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        if (*sorted == 0)\n\
            break;\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
    }\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        int4 data = read_imagei(input_t, coord.xy);\n\
        int4 index = read_imagei(indices_t, coord.xy);\n\
\n\
        write_imagei(output, coord.xy, data);\n\
        write_imagei(indices, coord.xy, index);\n\
    }\n\
}\n\
"; /* end of topk_odd_even_sort_cl*/

static const char topk_odd_even_sort2_cl[] = "#define LOCAL_SIZE_X    (32)\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X, 1, 1))) void topk_odd_even_sort_F32toU32_I32\n\
 (\n\
  __read_only  image2d_t input,\n\
               image2d_t input_t,\n\
               image2d_t indices_t,\n\
  __write_only image2d_t output,\n\
  __write_only image2d_t indices,\n\
               float     input_scale,\n\
               float     input_tail,\n\
               float     output_scale,\n\
               float     output_tail,\n\
               int       width\n\
  )\n\
 {\n\
    uint lid = get_local_id(0);\n\
    uint work_group_size = get_local_size(0);\n\
    uint offset = 0;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        float4 data = read_imagef(input, coord.xy);\n\
\n\
        write_imagef(input_t, coord.xy, data);\n\
        write_imagei(indices_t, coord.xy, coord.xxxx);\n\
    }\n\
\n\
    __local int sorted[1];\n\
\n\
    sorted[0] = 0;\n\
\n\
    while (1)\n\
    {\n\
        if (lid == 0)\n\
        {\n\
            *sorted = 0;\n\
        }\n\
        int swapped = 0;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        // odd-even\n\
        coord.x = lid * 2;\n\
        coord.z = lid * 2 + 1;\n\
        for (; coord.z < width; )\n\
        {\n\
            float4 left = read_imagef(input_t, coord.xy);\n\
            float4 right = read_imagef(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imagef(input_t, coord.xy, right);\n\
                write_imagef(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
        // even-odd\n\
        coord.x = lid * 2 + 1;\n\
        coord.z = lid * 2 + 2;\n\
        for (; coord.z < width; )\n\
        {\n\
            float4 left = read_imagef(input_t, coord.xy);\n\
            float4 right = read_imagef(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imagef(input_t, coord.xy, right);\n\
                write_imagef(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        atomic_add(sorted, swapped);\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        if (*sorted == 0)\n\
            break;\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
    }\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        float4 data = read_imagef(input_t, coord.xy);\n\
        int4 index = read_imagei(indices_t, coord.xy);\n\
\n\
        uint4 dst;\n\
        dst = convert_uint4(data * output_scale + output_tail);\n\
        write_imageui(output, coord.xy, dst);\n\
        write_imagei(indices, coord.xy, index);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X, 1, 1))) void topk_odd_even_sort_F32toI32_I32\n\
 (\n\
  __read_only  image2d_t input,\n\
               image2d_t input_t,\n\
               image2d_t indices_t,\n\
  __write_only image2d_t output,\n\
  __write_only image2d_t indices,\n\
               float     input_scale,\n\
               float     input_tail,\n\
               float     output_scale,\n\
               float     output_tail,\n\
               int       width\n\
  )\n\
 {\n\
    uint lid = get_local_id(0);\n\
    uint work_group_size = get_local_size(0);\n\
    uint offset = 0;\n\
\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        float4 data = read_imagef(input, coord.xy);\n\
\n\
        write_imagef(input_t, coord.xy, data);\n\
        write_imagei(indices_t, coord.xy, coord.xxxx);\n\
    }\n\
\n\
    __local int sorted[1];\n\
    sorted[0] = 0;\n\
\n\
    while (1)\n\
    {\n\
        if (lid == 0)\n\
        {\n\
            *sorted = 0;\n\
        }\n\
        int swapped = 0;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        // odd-even\n\
        coord.x = lid * 2;\n\
        coord.z = lid * 2 + 1;\n\
        for (; coord.z < width; )\n\
        {\n\
            float4 left = read_imagef(input_t, coord.xy);\n\
            float4 right = read_imagef(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                swapped = 1;\n\
\n\
                write_imagef(input_t, coord.xy, right);\n\
                write_imagef(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
        // even-odd\n\
        coord.x = lid * 2 + 1;\n\
        coord.z = lid * 2 + 2;\n\
        for (; coord.z < width; )\n\
        {\n\
            float4 left = read_imagef(input_t, coord.xy);\n\
            float4 right = read_imagef(input_t, coord.zy);\n\
            int4 l_index = read_imagei(indices_t, coord.xy);\n\
            int4 r_index = read_imagei(indices_t, coord.zy);\n\
\n\
            if ( (left.x < right.x) ||\n\
                (left.x == right.x && l_index.x < r_index.x) )\n\
            {\n\
                int4 l_index = read_imagei(indices_t, coord.xy);\n\
                int4 r_index = read_imagei(indices_t, coord.zy);\n\
                swapped = 1;\n\
\n\
                write_imagef(input_t, coord.xy, right);\n\
                write_imagef(input_t, coord.zy, left);\n\
\n\
                write_imagei(indices_t, coord.xy, r_index);\n\
                write_imagei(indices_t, coord.zy, l_index);\n\
            }\n\
\n\
            coord.xz += 2 * LOCAL_SIZE_X;\n\
        }\n\
\n\
        atomic_add(sorted, swapped);\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        if (*sorted == 0)\n\
            break;\n\
        barrier(CLK_GLOBAL_MEM_FENCE);\n\
    }\n\
\n\
    for (coord.x = lid; coord.x < width; coord.x += LOCAL_SIZE_X)\n\
    {\n\
        float4 data = read_imagef(input_t, coord.xy);\n\
        int4 index = read_imagei(indices_t, coord.xy);\n\
\n\
        int4 dst;\n\
        dst = convert_int4(data * output_scale + output_tail);\n\
        write_imagei(output, coord.xy, dst);\n\
        write_imagei(indices, coord.xy, index);\n\
    }\n\
}"; /* end of topk_odd_even_sort2_cl*/

static const char upsample_cl[] = "\n\
#define UPSAMPLE_PROCESS(data_type, read_fun, write_fun) \\\n\
    data_type src  = 0; \\\n\
    data_type dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    src.x  = read_fun(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    dst.x = axis.x == 0 ? src.x : 0; \\\n\
    write_fun(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? src.x : 0; \\\n\
    coord_out.x++; \\\n\
    write_fun(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? src.x : 0; \\\n\
    coord_out.y++; \\\n\
    write_fun(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? src.x : 0; \\\n\
    coord_out.x--; \\\n\
    write_fun(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_F32_U8to_F32(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_PROCESS(float4, read_imagef, write_imagef)\n\
}\n\
\n\
__kernel void upsample_F32_U8to_F32_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_PROCESS(float4, read_imagef, write_imagef)\n\
}\n\
\n\
__kernel void upsample_I32_U8to_I32(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_PROCESS(int4, read_imagei, write_imagei)\n\
}\n\
\n\
__kernel void upsample_I32_U8to_I32_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_PROCESS(int4, read_imagei, write_imagei)\n\
}\n\
\n\
\n\
#define UPSAMPLE_U8_PROCESS() \\\n\
    uint4  src  = 0; \\\n\
    uint4  dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    uint   output_zp = (uint)zp_out; \\\n\
    src.x  = read_imageui(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    result.x = convert_float4(src).x * scale_value + tail_value; \\\n\
    src = convert_uint4(result);\\\n\
    dst.x = axis.x == 0 ? src.x : output_zp; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? src.x : output_zp; \\\n\
    coord_out.x++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? src.x : output_zp; \\\n\
    coord_out.y++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? src.x : output_zp; \\\n\
    coord_out.x--; \\\n\
    write_imageui(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_U8_U8to_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output,\n\
                           float   scale_value,\n\
                           float   tail_value,\n\
                             int   zp_out)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_U8_PROCESS()\n\
}\n\
\n\
__kernel void upsample_U8_U8to_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output,\n\
                     float   scale_value,\n\
                     float   tail_value,\n\
                       int   zp_out)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_U8_PROCESS()\n\
}\n\
\n\
#define UPSAMPLE_U8_TO_F32PROCESS() \\\n\
    uint4  src  = 0; \\\n\
    float4  dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x  = read_imageui(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    result.x = convert_float4(src).x * scale_value + tail_value; \\\n\
    dst.x = axis.x == 0 ? result.x : 0.0f; \\\n\
    write_imagef(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? result.x : 0.0f; \\\n\
    coord_out.x++; \\\n\
    write_imagef(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? result.x : 0.0f; \\\n\
    coord_out.y++; \\\n\
    write_imagef(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? result.x : 0.0f; \\\n\
    coord_out.x--; \\\n\
    write_imagef(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_U8_U8to_F32(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output,\n\
                           float   scale_value,\n\
                           float   tail_value,\n\
                             int   zp_out)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_U8_TO_F32PROCESS()\n\
}\n\
\n\
__kernel void upsample_U8_U8to_F32_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output,\n\
                     float   scale_value,\n\
                     float   tail_value,\n\
                       int   zp_out)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_U8_TO_F32PROCESS()\n\
}\n\
\n\
\n\
#define UPSAMPLE_F32_TO_U8_PROCESS() \\\n\
    uint4  src  = 0; \\\n\
    uint4  dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    uint   output_zp = (uint)zp_out; \\\n\
    result.x  = read_imagef(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    result.x = result.x * scale_value + tail_value; \\\n\
    src = convert_uint4(result);\\\n\
    dst.x = axis.x == 0 ? src.x : output_zp; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? src.x : output_zp; \\\n\
    coord_out.x++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? src.x : output_zp; \\\n\
    coord_out.y++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? src.x : output_zp; \\\n\
    coord_out.x--; \\\n\
    write_imageui(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_F32_U8to_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output,\n\
                           float   scale_value,\n\
                           float   tail_value,\n\
                             int   zp_out)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_F32_TO_U8_PROCESS()\n\
}\n\
\n\
__kernel void upsample_F32_U8to_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output,\n\
                     float   scale_value,\n\
                     float   tail_value,\n\
                       int   zp_out)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_F32_TO_U8_PROCESS()\n\
}\n\
"; /* end of upsample_cl*/


typedef struct {
    char const* name;
    char const* data;
} source_map_t;

static const source_map_t evis_resource[] =
{
    {"a_times_b_plus_c_vx", a_times_b_plus_c_vx},
    {"add_mean_std_norm_vx", add_mean_std_norm_vx},
    {"argmax_axis0_vx", argmax_axis0_vx},
    {"argmax_axis1_vx", argmax_axis1_vx},
    {"argmax_axis2_vx", argmax_axis2_vx},
    {"argmin_axis0_vx", argmin_axis0_vx},
    {"argmin_axis1_vx", argmin_axis1_vx},
    {"argmin_axis2_vx", argmin_axis2_vx},
    {"batchnorm_single_vx", batchnorm_single_vx},
    {"batchnorm_single_f32_vx", batchnorm_single_f32_vx},
    {"bilinear_grid_sample_BF16_to_BF16_vx", bilinear_grid_sample_BF16_to_BF16_vx},
    {"bilinear_grid_sample_F16_to_F16_vx", bilinear_grid_sample_F16_to_F16_vx},
    {"bilinear_grid_sample_F16_to_U8_vx", bilinear_grid_sample_F16_to_U8_vx},
    {"bilinear_grid_sample_I16_to_I16_vx", bilinear_grid_sample_I16_to_I16_vx},
    {"bilinear_grid_sample_I8_to_I8_vx", bilinear_grid_sample_I8_to_I8_vx},
    {"bilinear_grid_sample_U8_to_U8_vx", bilinear_grid_sample_U8_to_U8_vx},
    {"bilinear_grid_sample_reflect_BF16_to_BF16_vx", bilinear_grid_sample_reflect_BF16_to_BF16_vx},
    {"bilinear_grid_sample_reflect_F16_to_F16_vx", bilinear_grid_sample_reflect_F16_to_F16_vx},
    {"bilinear_grid_sample_reflect_F16_to_U8_vx", bilinear_grid_sample_reflect_F16_to_U8_vx},
    {"bilinear_grid_sample_reflect_I16_to_I16_vx", bilinear_grid_sample_reflect_I16_to_I16_vx},
    {"bilinear_grid_sample_reflect_I8_to_I8_vx", bilinear_grid_sample_reflect_I8_to_I8_vx},
    {"bilinear_grid_sample_reflect_U8_to_U8_vx", bilinear_grid_sample_reflect_U8_to_U8_vx},
    {"bucketize_vx", bucketize_vx},
    {"cast_vx", cast_vx},
    {"clip_F16_vx", clip_F16_vx},
    {"clip_I16_vx", clip_I16_vx},
    {"clip_I8_vx", clip_I8_vx},
    {"clip_U8_vx", clip_U8_vx},
    {"conv1d_ovxlib_vx", conv1d_ovxlib_vx},
    {"conv1d_ovxlib_k1024_vx", conv1d_ovxlib_k1024_vx},
    {"crop_and_resize_bilinear_vx", crop_and_resize_bilinear_vx},
    {"crop_and_resize_nearest_neighbor_vx", crop_and_resize_nearest_neighbor_vx},
    {"cumsum_vx", cumsum_vx},
    {"cumsum_2d_vx", cumsum_2d_vx},
    {"cumsum_array_vx", cumsum_array_vx},
    {"cumsum_array_2d_vx", cumsum_array_2d_vx},
    {"cumsum_array_bf16_vx", cumsum_array_bf16_vx},
    {"cumsum_array_ex_rev_axis0_vx", cumsum_array_ex_rev_axis0_vx},
    {"cumsum_array_ex_rev_axis1_vx", cumsum_array_ex_rev_axis1_vx},
    {"cumsum_array_ex_rev_axis2_vx", cumsum_array_ex_rev_axis2_vx},
    {"cumsum_array_f16_u8_vx", cumsum_array_f16_u8_vx},
    {"cumsum_array_f16_u8_2d_vx", cumsum_array_f16_u8_2d_vx},
    {"cumsum_bf16_vx", cumsum_bf16_vx},
    {"cumsum_ex_rev_axis0_vx", cumsum_ex_rev_axis0_vx},
    {"cumsum_ex_rev_axis1_vx", cumsum_ex_rev_axis1_vx},
    {"cumsum_ex_rev_axis2_vx", cumsum_ex_rev_axis2_vx},
    {"cumsum_f16_u8_vx", cumsum_f16_u8_vx},
    {"custom_letterbox_vx", custom_letterbox_vx},
    {"custom_softmax_vx", custom_softmax_vx},
    {"custom_warp_affine_vx", custom_warp_affine_vx},
    {"custom_warp_affine_2d_vx", custom_warp_affine_2d_vx},
    {"custom_warp_affine_optional_vx", custom_warp_affine_optional_vx},
    {"custom_warp_affine_rgb_vx", custom_warp_affine_rgb_vx},
    {"custom_warp_affine_rgb_optional_vx", custom_warp_affine_rgb_optional_vx},
    {"custom_warp_perspective_vx", custom_warp_perspective_vx},
    {"depth2space_crd_vx", depth2space_crd_vx},
    {"depthwise_conv1d_src0_vx", depthwise_conv1d_src0_vx},
    {"depthwise_conv1d_src1_vx", depthwise_conv1d_src1_vx},
    {"depthwise_conv1d_src2_vx", depthwise_conv1d_src2_vx},
    {"depthwise_conv1d_src3_vx", depthwise_conv1d_src3_vx},
    {"detect_post_box_vx", detect_post_box_vx},
    {"eltwise_unary_2d_0_vx", eltwise_unary_2d_0_vx},
    {"eltwise_unary_2d_1_vx", eltwise_unary_2d_1_vx},
    {"eltwise_unary_3d_0_vx", eltwise_unary_3d_0_vx},
    {"eltwise_unary_3d_1_vx", eltwise_unary_3d_1_vx},
    {"erf_vx", erf_vx},
    {"extra_ending_vx", extra_ending_vx},
    {"floordiv_vx", floordiv_vx},
    {"gather_vx", gather_vx},
    {"gather_array_vx", gather_array_vx},
    {"gather_batch_vx", gather_batch_vx},
    {"gather_elements_vx", gather_elements_vx},
    {"gather_mix_vx", gather_mix_vx},
    {"gather_mix_batch_vx", gather_mix_batch_vx},
    {"gather_nd_vx", gather_nd_vx},
    {"gather_nd_2d_vx", gather_nd_2d_vx},
    {"gather_nd_2d_mix_vx", gather_nd_2d_mix_vx},
    {"gather_nd_3d_vx", gather_nd_3d_vx},
    {"gather_nd_3d_mix_vx", gather_nd_3d_mix_vx},
    {"gather_nd_batch_vx", gather_nd_batch_vx},
    {"gather_nd_batch_2d_vx", gather_nd_batch_2d_vx},
    {"gather_nd_mix_vx", gather_nd_mix_vx},
    {"get_matrix_vx", get_matrix_vx},
    {"group_normalization_0_vx", group_normalization_0_vx},
    {"group_normalization_1_vx", group_normalization_1_vx},
    {"group_normalization_2_vx", group_normalization_2_vx},
    {"grucell_activation_vx", grucell_activation_vx},
    {"grucell_activation_sma_vx", grucell_activation_sma_vx},
    {"grucell_activation_z_h_vx", grucell_activation_z_h_vx},
    {"grucell_cdnn_activation_vx", grucell_cdnn_activation_vx},
    {"grucell_cdnn_activation_bf16_vx", grucell_cdnn_activation_bf16_vx},
    {"grucell_cdnn_activation_u8_vx", grucell_cdnn_activation_u8_vx},
    {"grucell_h_times_activation_r_vx", grucell_h_times_activation_r_vx},
    {"grucell_reset_after_activation_vx", grucell_reset_after_activation_vx},
    {"hswish_vx", hswish_vx},
    {"instance_normalization_0_vx", instance_normalization_0_vx},
    {"instance_normalization_1_vx", instance_normalization_1_vx},
    {"instance_normalization_2_vx", instance_normalization_2_vx},
    {"instance_normalization_3_vx", instance_normalization_3_vx},
    {"l1norm_axis0_vx", l1norm_axis0_vx},
    {"l1norm_axis1_vx", l1norm_axis1_vx},
    {"l1norm_axis2_vx", l1norm_axis2_vx},
    {"l2normalizescale_axis0_vx", l2normalizescale_axis0_vx},
    {"l2normalizescale_axis0_2d_vx", l2normalizescale_axis0_2d_vx},
    {"l2normalizescale_axis1_vx", l2normalizescale_axis1_vx},
    {"layer_normalization_0_vx", layer_normalization_0_vx},
    {"layer_normalization_1_vx", layer_normalization_1_vx},
    {"layer_normalization_2_vx", layer_normalization_2_vx},
    {"layer_normalization_3_vx", layer_normalization_3_vx},
    {"layer_normalization_axis01_0_vx", layer_normalization_axis01_0_vx},
    {"layer_normalization_axis01_1_vx", layer_normalization_axis01_1_vx},
    {"layer_normalization_axis01_2_vx", layer_normalization_axis01_2_vx},
    {"layer_normalization_axis01_3_vx", layer_normalization_axis01_3_vx},
    {"layer_normalization_axis01_sum_vx", layer_normalization_axis01_sum_vx},
    {"log_softmax_axis0_vx", log_softmax_axis0_vx},
    {"log_softmax_axis0_BF16_vx", log_softmax_axis0_BF16_vx},
    {"log_softmax_axis1_vx", log_softmax_axis1_vx},
    {"log_softmax_axis1_BF16_vx", log_softmax_axis1_BF16_vx},
    {"log_softmax_axis2_vx", log_softmax_axis2_vx},
    {"log_softmax_exceed_axis0_vx", log_softmax_exceed_axis0_vx},
    {"log_softmax_exceed_axis0_BF16_vx", log_softmax_exceed_axis0_BF16_vx},
    {"log_softmax_exceed_axis1_vx", log_softmax_exceed_axis1_vx},
    {"log_softmax_exceed_axis1_BF16_vx", log_softmax_exceed_axis1_BF16_vx},
    {"logical_not_vx", logical_not_vx},
    {"logical_ops_vx", logical_ops_vx},
    {"lstmunit_activation_BP_BF16_vx", lstmunit_activation_BP_BF16_vx},
    {"lstmunit_activation_BP_F16_vx", lstmunit_activation_BP_F16_vx},
    {"lstmunit_activation_BP_U8_vx", lstmunit_activation_BP_U8_vx},
    {"lstmunit_activation_B_BF16_vx", lstmunit_activation_B_BF16_vx},
    {"lstmunit_activation_B_F16_vx", lstmunit_activation_B_F16_vx},
    {"lstmunit_activation_B_U8_vx", lstmunit_activation_B_U8_vx},
    {"lstmunit_activation_CBP_BF16_vx", lstmunit_activation_CBP_BF16_vx},
    {"lstmunit_activation_CBP_F16_vx", lstmunit_activation_CBP_F16_vx},
    {"lstmunit_activation_CBP_U8_vx", lstmunit_activation_CBP_U8_vx},
    {"lstmunit_activation_CB_BF16_vx", lstmunit_activation_CB_BF16_vx},
    {"lstmunit_activation_CB_F16_vx", lstmunit_activation_CB_F16_vx},
    {"lstmunit_activation_CB_U8_vx", lstmunit_activation_CB_U8_vx},
    {"lstmunit_activation_CLP_BF16_vx", lstmunit_activation_CLP_BF16_vx},
    {"lstmunit_activation_CLP_F16_vx", lstmunit_activation_CLP_F16_vx},
    {"lstmunit_activation_CL_BF16_vx", lstmunit_activation_CL_BF16_vx},
    {"lstmunit_activation_CL_F16_vx", lstmunit_activation_CL_F16_vx},
    {"lstmunit_activation_CSP_BF16_vx", lstmunit_activation_CSP_BF16_vx},
    {"lstmunit_activation_CSP_F16_vx", lstmunit_activation_CSP_F16_vx},
    {"lstmunit_activation_CSP_U8_vx", lstmunit_activation_CSP_U8_vx},
    {"lstmunit_activation_CS_BF16_vx", lstmunit_activation_CS_BF16_vx},
    {"lstmunit_activation_CS_F16_vx", lstmunit_activation_CS_F16_vx},
    {"lstmunit_activation_CS_U8_vx", lstmunit_activation_CS_U8_vx},
    {"lstmunit_activation_LP_BF16_vx", lstmunit_activation_LP_BF16_vx},
    {"lstmunit_activation_LP_F16_vx", lstmunit_activation_LP_F16_vx},
    {"lstmunit_activation_L_BF16_vx", lstmunit_activation_L_BF16_vx},
    {"lstmunit_activation_L_F16_vx", lstmunit_activation_L_F16_vx},
    {"lstmunit_activation_SP_BF16_vx", lstmunit_activation_SP_BF16_vx},
    {"lstmunit_activation_SP_F16_vx", lstmunit_activation_SP_F16_vx},
    {"lstmunit_activation_SP_U8_vx", lstmunit_activation_SP_U8_vx},
    {"lstmunit_activation_S_BF16_vx", lstmunit_activation_S_BF16_vx},
    {"lstmunit_activation_S_F16_vx", lstmunit_activation_S_F16_vx},
    {"lstmunit_activation_S_U8_vx", lstmunit_activation_S_U8_vx},
    {"matrixmul_bf16_vx", matrixmul_bf16_vx},
    {"matrixmul_cross_vx", matrixmul_cross_vx},
    {"matrixmul_cross_i16_vx", matrixmul_cross_i16_vx},
    {"matrixmul_f16_vx", matrixmul_f16_vx},
    {"matrixmul_f16f16_u8_vx", matrixmul_f16f16_u8_vx},
    {"matrixmul_f16i16_i16_vx", matrixmul_f16i16_i16_vx},
    {"matrixmul_f16u8_f16_vx", matrixmul_f16u8_f16_vx},
    {"matrixmul_f16u8_u8_vx", matrixmul_f16u8_u8_vx},
    {"matrixmul_i16_vx", matrixmul_i16_vx},
    {"matrixmul_merge_vx", matrixmul_merge_vx},
    {"matrixmul_transA_vx", matrixmul_transA_vx},
    {"matrixmul_transB_f16_vx", matrixmul_transB_f16_vx},
    {"matrixmul_transB_f16_mix_vx", matrixmul_transB_f16_mix_vx},
    {"matrixmul_transB_u8_mix_vx", matrixmul_transB_u8_mix_vx},
    {"matrixmul_u8_vx", matrixmul_u8_vx},
    {"matrixmul_u8f16_f16_vx", matrixmul_u8f16_f16_vx},
    {"matrixmul_u8f16_u8_vx", matrixmul_u8f16_u8_vx},
    {"matrixmul_u8i16_i16_vx", matrixmul_u8i16_i16_vx},
    {"matrixmul_u8u8_f16_vx", matrixmul_u8u8_f16_vx},
    {"maximum_0_vx", maximum_0_vx},
    {"maximum_1_vx", maximum_1_vx},
    {"maxpool_vx", maxpool_vx},
    {"minimum_0_vx", minimum_0_vx},
    {"minimum_1_vx", minimum_1_vx},
    {"mod_vx", mod_vx},
    {"moments_axis0_vx", moments_axis0_vx},
    {"moments_axis01_vx", moments_axis01_vx},
    {"moments_axis012_vx", moments_axis012_vx},
    {"moments_axis1_vx", moments_axis1_vx},
    {"moments_axis2_vx", moments_axis2_vx},
    {"moments_u8_vx", moments_u8_vx},
    {"moments_u8_axis012_vx", moments_u8_axis012_vx},
    {"nearest_grid_sample_BF16_to_BF16_vx", nearest_grid_sample_BF16_to_BF16_vx},
    {"nearest_grid_sample_F16_to_F16_vx", nearest_grid_sample_F16_to_F16_vx},
    {"nearest_grid_sample_F16_to_U8_vx", nearest_grid_sample_F16_to_U8_vx},
    {"nearest_grid_sample_I16_to_I16_vx", nearest_grid_sample_I16_to_I16_vx},
    {"nearest_grid_sample_I8_to_I8_vx", nearest_grid_sample_I8_to_I8_vx},
    {"nearest_grid_sample_U8_to_U8_vx", nearest_grid_sample_U8_to_U8_vx},
    {"one_hot_vx", one_hot_vx},
    {"poolwithargmax_F16_vx", poolwithargmax_F16_vx},
    {"poolwithargmax_I16_vx", poolwithargmax_I16_vx},
    {"poolwithargmax_I8_vx", poolwithargmax_I8_vx},
    {"poolwithargmax_U8_vx", poolwithargmax_U8_vx},
    {"pow_vx", pow_vx},
    {"pre_process_bgra_vx", pre_process_bgra_vx},
    {"pre_process_gray_vx", pre_process_gray_vx},
    {"pre_process_gray_2_vx", pre_process_gray_2_vx},
    {"pre_process_gray_copy_vx", pre_process_gray_copy_vx},
    {"pre_process_nv12_copy_vx", pre_process_nv12_copy_vx},
    {"pre_process_nv12_rggb_copy_vx", pre_process_nv12_rggb_copy_vx},
    {"pre_process_nv12_rggb_scale_vx", pre_process_nv12_rggb_scale_vx},
    {"pre_process_nv12_scale_vx", pre_process_nv12_scale_vx},
    {"pre_process_rgb_vx", pre_process_rgb_vx},
    {"pre_process_rgb888_planar_0_vx", pre_process_rgb888_planar_0_vx},
    {"pre_process_rgb888_planar_1_vx", pre_process_rgb888_planar_1_vx},
    {"pre_process_rgb888_planar_2_vx", pre_process_rgb888_planar_2_vx},
    {"pre_process_rgb888_planar_nhwc_0_vx", pre_process_rgb888_planar_nhwc_0_vx},
    {"pre_process_rgb888_planar_nhwc_1_vx", pre_process_rgb888_planar_nhwc_1_vx},
    {"pre_process_rgb888_planar_nhwc_2_vx", pre_process_rgb888_planar_nhwc_2_vx},
    {"pre_process_rgb888_planar_sep_0_vx", pre_process_rgb888_planar_sep_0_vx},
    {"pre_process_rgb888_planar_sep_1_vx", pre_process_rgb888_planar_sep_1_vx},
    {"pre_process_rgb888_planar_sep_2_vx", pre_process_rgb888_planar_sep_2_vx},
    {"pre_process_rgb888_planar_sep_nhwc_0_vx", pre_process_rgb888_planar_sep_nhwc_0_vx},
    {"pre_process_rgb888_planar_sep_nhwc_1_vx", pre_process_rgb888_planar_sep_nhwc_1_vx},
    {"pre_process_rgb888_planar_sep_nhwc_2_vx", pre_process_rgb888_planar_sep_nhwc_2_vx},
    {"pre_process_rgb_copy_vx", pre_process_rgb_copy_vx},
    {"pre_process_yuv420_copy_vx", pre_process_yuv420_copy_vx},
    {"pre_process_yuv420_scale_0_vx", pre_process_yuv420_scale_0_vx},
    {"pre_process_yuv420_scale_1_vx", pre_process_yuv420_scale_1_vx},
    {"pre_process_yuv422_copy_vx", pre_process_yuv422_copy_vx},
    {"pre_process_yuv422_scale_vx", pre_process_yuv422_scale_vx},
    {"pre_process_yuv444_copy_u8_vx", pre_process_yuv444_copy_u8_vx},
    {"pre_process_yuv444_scale_vx", pre_process_yuv444_scale_vx},
    {"pre_process_yuv444_scale_fp16_vx", pre_process_yuv444_scale_fp16_vx},
    {"prelu_vx", prelu_vx},
    {"prelu_BF16_vx", prelu_BF16_vx},
    {"random_multinomial_vx", random_multinomial_vx},
    {"reduceall_internal_axis0_vx", reduceall_internal_axis0_vx},
    {"reduceall_internal_axis1_vx", reduceall_internal_axis1_vx},
    {"reduceall_internal_axis2_vx", reduceall_internal_axis2_vx},
    {"reduceany_internal_axis0_vx", reduceany_internal_axis0_vx},
    {"reduceany_internal_axis1_vx", reduceany_internal_axis1_vx},
    {"reduceany_internal_axis2_vx", reduceany_internal_axis2_vx},
    {"reducemax_internal_axis0_vx", reducemax_internal_axis0_vx},
    {"reducemax_internal_axis1_vx", reducemax_internal_axis1_vx},
    {"reducemax_internal_axis2_vx", reducemax_internal_axis2_vx},
    {"reducemin_internal_axis0_vx", reducemin_internal_axis0_vx},
    {"reducemin_internal_axis1_vx", reducemin_internal_axis1_vx},
    {"reducemin_internal_axis2_vx", reducemin_internal_axis2_vx},
    {"reduceprod_internal_axis0_vx", reduceprod_internal_axis0_vx},
    {"reduceprod_internal_axis1_vx", reduceprod_internal_axis1_vx},
    {"reduceprod_internal_axis2_vx", reduceprod_internal_axis2_vx},
    {"relational_ops_2d_vx", relational_ops_2d_vx},
    {"relational_ops_3d_vx", relational_ops_3d_vx},
    {"relu_keras_vx", relu_keras_vx},
    {"repeat_vx", repeat_vx},
    {"repeat_axis1_vx", repeat_axis1_vx},
    {"resize_1d_bilinear_BF16_vx", resize_1d_bilinear_BF16_vx},
    {"resize_1d_bilinear_DOWN_NX_vx", resize_1d_bilinear_DOWN_NX_vx},
    {"resize_1d_bilinear_F16_vx", resize_1d_bilinear_F16_vx},
    {"resize_1d_bilinear_I16_vx", resize_1d_bilinear_I16_vx},
    {"resize_1d_bilinear_I8_vx", resize_1d_bilinear_I8_vx},
    {"resize_1d_bilinear_U8_vx", resize_1d_bilinear_U8_vx},
    {"resize_1d_bilinear_U8_opt_vx", resize_1d_bilinear_U8_opt_vx},
    {"resize_1d_bilinear_UP_NX_vx", resize_1d_bilinear_UP_NX_vx},
    {"resize_1d_nearest_vx", resize_1d_nearest_vx},
    {"resize_bilinear_BF16_vx", resize_bilinear_BF16_vx},
    {"resize_bilinear_F16_vx", resize_bilinear_F16_vx},
    {"resize_bilinear_I16_vx", resize_bilinear_I16_vx},
    {"resize_bilinear_I8_vx", resize_bilinear_I8_vx},
    {"resize_bilinear_U16_vx", resize_bilinear_U16_vx},
    {"resize_bilinear_U8_vx", resize_bilinear_U8_vx},
    {"resize_bilinear_U8_half_pixel_centers_1_vx", resize_bilinear_U8_half_pixel_centers_1_vx},
    {"resize_bilinear_U8_half_pixel_centers_2_vx", resize_bilinear_U8_half_pixel_centers_2_vx},
    {"resize_bilinear_U8_half_pixel_centers_3_vx", resize_bilinear_U8_half_pixel_centers_3_vx},
    {"resize_bilinear_U8_half_pixel_centers_4_vx", resize_bilinear_U8_half_pixel_centers_4_vx},
    {"resize_bilinear_U8_half_pixel_centers_5_vx", resize_bilinear_U8_half_pixel_centers_5_vx},
    {"resize_bilinear_U8_opt_vx", resize_bilinear_U8_opt_vx},
    {"resize_bilinear_align_corners_vx", resize_bilinear_align_corners_vx},
    {"resize_bilinear_nhwc_vx", resize_bilinear_nhwc_vx},
    {"resize_bilinear_nhwc_bound_vx", resize_bilinear_nhwc_bound_vx},
    {"resize_cubic_vx", resize_cubic_vx},
    {"resize_nearest_vx", resize_nearest_vx},
    {"rope_0_vx", rope_0_vx},
    {"rope_1_vx", rope_1_vx},
    {"rope_2_vx", rope_2_vx},
    {"rope_3_vx", rope_3_vx},
    {"scatter_nd_vx", scatter_nd_vx},
    {"scatter_nd_big_vx", scatter_nd_big_vx},
    {"scatter_nd_update_vx", scatter_nd_update_vx},
    {"scatter_nd_update_atom_vx", scatter_nd_update_atom_vx},
    {"scatter_nd_update_big_vx", scatter_nd_update_big_vx},
    {"scatter_nd_update_fp_vx", scatter_nd_update_fp_vx},
    {"scatter_nd_update_qint_vx", scatter_nd_update_qint_vx},
    {"scatter_nd_update_reduction_vx", scatter_nd_update_reduction_vx},
    {"scatter_nd_update_reduction_conv_vx", scatter_nd_update_reduction_conv_vx},
    {"scatter_nd_update_special_vx", scatter_nd_update_special_vx},
    {"select_vx", select_vx},
    {"sequence_mask_vx", sequence_mask_vx},
    {"signal_frame_vx", signal_frame_vx},
    {"slice_vx", slice_vx},
    {"space2depth_internal_vx", space2depth_internal_vx},
    {"swish_vx", swish_vx},
    {"tensorstackconcat_vx", tensorstackconcat_vx},
    {"tile_vx", tile_vx},
    {"tile_mix_vx", tile_mix_vx},
    {"tiny_yolov4_postprocess_box_vx", tiny_yolov4_postprocess_box_vx},
    {"tiny_yolov4_postprocess_confidence_vx", tiny_yolov4_postprocess_confidence_vx},
    {"upsample_F16_vx", upsample_F16_vx},
    {"upsample_I16_vx", upsample_I16_vx},
    {"upsample_I8_vx", upsample_I8_vx},
    {"upsample_U8_vx", upsample_U8_vx},
    {"upsamplescale_vx", upsamplescale_vx},
    {"upsamplescale_k2_vx", upsamplescale_k2_vx},
    {"vsi_nn_kernel_header_vx", vsi_nn_kernel_header_vx},
    {"warp_affine_vx", warp_affine_vx},
};

static const source_map_t cl_resource[] =
{
    {"add_mean_std_norm_cl", add_mean_std_norm_cl},
    {"argmax_axis0_cl", argmax_axis0_cl},
    {"argmax_axis1_cl", argmax_axis1_cl},
    {"argmax_axis2_cl", argmax_axis2_cl},
    {"argmin_axis0_cl", argmin_axis0_cl},
    {"argmin_axis1_cl", argmin_axis1_cl},
    {"argmin_axis2_cl", argmin_axis2_cl},
    {"avg_pool3d_cl", avg_pool3d_cl},
    {"batchnorm_single_cl", batchnorm_single_cl},
    {"bilinear_grid_sample_cl", bilinear_grid_sample_cl},
    {"bilinear_grid_sample_reflect_cl", bilinear_grid_sample_reflect_cl},
    {"bucketize_cl", bucketize_cl},
    {"cast_cl", cast_cl},
    {"clip_BF16_cl", clip_BF16_cl},
    {"clip_F32_cl", clip_F32_cl},
    {"clip_I32_cl", clip_I32_cl},
    {"clip_U8_cl", clip_U8_cl},
    {"col2im_cl", col2im_cl},
    {"crop_and_resize_bilinear_cl", crop_and_resize_bilinear_cl},
    {"crop_and_resize_nearest_neighbor_cl", crop_and_resize_nearest_neighbor_cl},
    {"cumsum_cl", cumsum_cl},
    {"cumsum_2d_cl", cumsum_2d_cl},
    {"cumsum_array_2d_axis0_cl", cumsum_array_2d_axis0_cl},
    {"cumsum_array_2d_axis1_cl", cumsum_array_2d_axis1_cl},
    {"cumsum_array_axis0_cl", cumsum_array_axis0_cl},
    {"cumsum_array_axis1_cl", cumsum_array_axis1_cl},
    {"cumsum_array_axis2_cl", cumsum_array_axis2_cl},
    {"depth2space_crd_cl", depth2space_crd_cl},
    {"eltwise_ops_helper_cl", eltwise_ops_helper_cl},
    {"eltwise_unary_0_cl", eltwise_unary_0_cl},
    {"eltwise_unary_1_cl", eltwise_unary_1_cl},
    {"erf_cl", erf_cl},
    {"floordiv_cl", floordiv_cl},
    {"gather_cl", gather_cl},
    {"gather_array_cl", gather_array_cl},
    {"gather_batch_cl", gather_batch_cl},
    {"gather_elements_cl", gather_elements_cl},
    {"gather_nd_cl", gather_nd_cl},
    {"gather_nd_3d_cl", gather_nd_3d_cl},
    {"gather_nd_batch_cl", gather_nd_batch_cl},
    {"globallppool_cl", globallppool_cl},
    {"group_normalization_f32_cl", group_normalization_f32_cl},
    {"group_normalization_i32_cl", group_normalization_i32_cl},
    {"group_normalization_u8_cl", group_normalization_u8_cl},
    {"grucell_activation_cl", grucell_activation_cl},
    {"grucell_activation_sma_cl", grucell_activation_sma_cl},
    {"grucell_activation_z_h_cl", grucell_activation_z_h_cl},
    {"grucell_h_times_activation_r_cl", grucell_h_times_activation_r_cl},
    {"grucell_reset_after_activation_cl", grucell_reset_after_activation_cl},
    {"hswish_cl", hswish_cl},
    {"instance_normalization_f32_cl", instance_normalization_f32_cl},
    {"instance_normalization_i32_cl", instance_normalization_i32_cl},
    {"instance_normalization_u8_cl", instance_normalization_u8_cl},
    {"l1norm_cl", l1norm_cl},
    {"l2normalizescale_axis0_cl", l2normalizescale_axis0_cl},
    {"l2normalizescale_axis1_cl", l2normalizescale_axis1_cl},
    {"layer_normalization_cl", layer_normalization_cl},
    {"log_softmax_axis0_cl", log_softmax_axis0_cl},
    {"log_softmax_axis1_cl", log_softmax_axis1_cl},
    {"log_softmax_axis2_cl", log_softmax_axis2_cl},
    {"log_softmax_exceed_axis0_cl", log_softmax_exceed_axis0_cl},
    {"log_softmax_exceed_axis1_cl", log_softmax_exceed_axis1_cl},
    {"logical_not_cl", logical_not_cl},
    {"logical_ops_cl", logical_ops_cl},
    {"lppool_cl", lppool_cl},
    {"lstmunit_activation_BP_F32_cl", lstmunit_activation_BP_F32_cl},
    {"lstmunit_activation_BP_U8_cl", lstmunit_activation_BP_U8_cl},
    {"lstmunit_activation_B_F32_cl", lstmunit_activation_B_F32_cl},
    {"lstmunit_activation_B_U8_cl", lstmunit_activation_B_U8_cl},
    {"lstmunit_activation_CBP_F32_cl", lstmunit_activation_CBP_F32_cl},
    {"lstmunit_activation_CBP_U8_cl", lstmunit_activation_CBP_U8_cl},
    {"lstmunit_activation_CB_F32_cl", lstmunit_activation_CB_F32_cl},
    {"lstmunit_activation_CB_U8_cl", lstmunit_activation_CB_U8_cl},
    {"lstmunit_activation_CLP_F32_cl", lstmunit_activation_CLP_F32_cl},
    {"lstmunit_activation_CLP_U8_cl", lstmunit_activation_CLP_U8_cl},
    {"lstmunit_activation_CL_F32_cl", lstmunit_activation_CL_F32_cl},
    {"lstmunit_activation_CL_U8_cl", lstmunit_activation_CL_U8_cl},
    {"lstmunit_activation_CSP_F32_cl", lstmunit_activation_CSP_F32_cl},
    {"lstmunit_activation_CSP_U8_cl", lstmunit_activation_CSP_U8_cl},
    {"lstmunit_activation_CS_F32_cl", lstmunit_activation_CS_F32_cl},
    {"lstmunit_activation_CS_U8_cl", lstmunit_activation_CS_U8_cl},
    {"lstmunit_activation_LP_F32_cl", lstmunit_activation_LP_F32_cl},
    {"lstmunit_activation_L_F32_cl", lstmunit_activation_L_F32_cl},
    {"lstmunit_activation_SP_F32_cl", lstmunit_activation_SP_F32_cl},
    {"lstmunit_activation_SP_U8_cl", lstmunit_activation_SP_U8_cl},
    {"lstmunit_activation_S_F32_cl", lstmunit_activation_S_F32_cl},
    {"lstmunit_activation_S_U8_cl", lstmunit_activation_S_U8_cl},
    {"matrixmul_cl", matrixmul_cl},
    {"matrixmul_4x_cl", matrixmul_4x_cl},
    {"matrixmul_cross_cl", matrixmul_cross_cl},
    {"matrixmul_transA_cl", matrixmul_transA_cl},
    {"maximum_cl", maximum_cl},
    {"maxpool_cl", maxpool_cl},
    {"maxpoolwithargmax_cl", maxpoolwithargmax_cl},
    {"maxpoolwithargmax_2d_cl", maxpoolwithargmax_2d_cl},
    {"maxunpool_cl", maxunpool_cl},
    {"minimum_cl", minimum_cl},
    {"mod_cl", mod_cl},
    {"moments_axis0_cl", moments_axis0_cl},
    {"moments_axis01_cl", moments_axis01_cl},
    {"moments_axis012_cl", moments_axis012_cl},
    {"moments_axis1_cl", moments_axis1_cl},
    {"moments_axis2_cl", moments_axis2_cl},
    {"nearest_grid_sample_cl", nearest_grid_sample_cl},
    {"one_hot_cl", one_hot_cl},
    {"poolwithargmax_cl", poolwithargmax_cl},
    {"pow_cl", pow_cl},
    {"prelu_cl", prelu_cl},
    {"random_multinomial_cl", random_multinomial_cl},
    {"reduceall_internal_axis0_cl", reduceall_internal_axis0_cl},
    {"reduceall_internal_axis1_cl", reduceall_internal_axis1_cl},
    {"reduceall_internal_axis2_cl", reduceall_internal_axis2_cl},
    {"reduceany_internal_axis0_cl", reduceany_internal_axis0_cl},
    {"reduceany_internal_axis1_cl", reduceany_internal_axis1_cl},
    {"reduceany_internal_axis2_cl", reduceany_internal_axis2_cl},
    {"reducemax_internal_axis0_cl", reducemax_internal_axis0_cl},
    {"reducemax_internal_axis1_cl", reducemax_internal_axis1_cl},
    {"reducemax_internal_axis2_cl", reducemax_internal_axis2_cl},
    {"reducemin_internal_axis0_cl", reducemin_internal_axis0_cl},
    {"reducemin_internal_axis1_cl", reducemin_internal_axis1_cl},
    {"reducemin_internal_axis2_cl", reducemin_internal_axis2_cl},
    {"reduceprod_internal_axis0_cl", reduceprod_internal_axis0_cl},
    {"reduceprod_internal_axis1_cl", reduceprod_internal_axis1_cl},
    {"reduceprod_internal_axis2_cl", reduceprod_internal_axis2_cl},
    {"relational_ops_cl", relational_ops_cl},
    {"relu_keras_cl", relu_keras_cl},
    {"repeat_cl", repeat_cl},
    {"resize_1d_bilinear_cl", resize_1d_bilinear_cl},
    {"resize_1d_nearest_cl", resize_1d_nearest_cl},
    {"resize_3d_bilinear_cl", resize_3d_bilinear_cl},
    {"resize_3d_nearest_cl", resize_3d_nearest_cl},
    {"resize_bilinear_cl", resize_bilinear_cl},
    {"resize_cubic_cl", resize_cubic_cl},
    {"resize_nearest_cl", resize_nearest_cl},
    {"reversesequence_cl", reversesequence_cl},
    {"roi_align_cl", roi_align_cl},
    {"rope_0_cl", rope_0_cl},
    {"scatter_elements_cl", scatter_elements_cl},
    {"scatter_elements_add_cl", scatter_elements_add_cl},
    {"scatter_elements_mul_cl", scatter_elements_mul_cl},
    {"scatter_nd_cl", scatter_nd_cl},
    {"scatter_nd_update_cl", scatter_nd_update_cl},
    {"scatter_nd_update_reduction_cl", scatter_nd_update_reduction_cl},
    {"scatter_nd_update_reduction_conv_cl", scatter_nd_update_reduction_conv_cl},
    {"select_cl", select_cl},
    {"sequence_mask_cl", sequence_mask_cl},
    {"signal_frame_cl", signal_frame_cl},
    {"slice_cl", slice_cl},
    {"space2depth_internal_cl", space2depth_internal_cl},
    {"swish_cl", swish_cl},
    {"tile_cl", tile_cl},
    {"topk_cl", topk_cl},
    {"topk2_cl", topk2_cl},
    {"topk_odd_even_sort_cl", topk_odd_even_sort_cl},
    {"topk_odd_even_sort2_cl", topk_odd_even_sort2_cl},
    {"upsample_cl", upsample_cl},
};

static const char* _load_code
    (
    const char* source_name,
    size_t* size,
    const source_map_t* source_map,
    size_t source_map_size,
    const char* tail
    )
{
    const char* source;
    char source_path[VSI_NN_MAX_PATH];
    size_t n;
    int i;
    source = NULL;
    n = snprintf( source_path, VSI_NN_MAX_PATH, "%s%s", source_name, tail );
    if( n == VSI_NN_MAX_PATH )
    {
        VSILOGE("Kernel source path overflow %d/%d", n, VSI_NN_MAX_PATH);
        *size = 0;
        return NULL;
    }
    for( i = 0; i < (int)source_map_size; i++ )
    {
        if( strncmp( source_map[i].name, source_path, VSI_NN_MAX_PATH ) == 0 )
        {
            source = source_map[i].data;
            *size = strlen( source );
            break;
        }
    }
    if( !source )
    {
        *size = 0;
    }
    return source;
} /* _load_code() */

const char* vsi_nn_resource_load_source_code
    (
    const char* source_name,
    size_t* size,
    vsi_nn_kernel_type_e type
    )
{
    const char* s = NULL;
    switch( type )
    {
        case VSI_NN_KERNEL_TYPE_EVIS:
            s = _load_code( source_name, size,
                evis_resource, _cnt_of_array(evis_resource), "_vx" );
            break;
        case VSI_NN_KERNEL_TYPE_CL:
            s = _load_code( source_name, size,
                cl_resource, _cnt_of_array(cl_resource), "_cl" );
            break;
        default:
            break;
    }
    return s;
} /* vsi_nn_resource_load_source_code() */
