/****************************************************************************
*
*    Copyright (c) 2020 Vivante Corporation
*
*    Permission is hereby granted, free of charge, to any person obtaining a
*    copy of this software and associated documentation files (the "Software"),
*    to deal in the Software without restriction, including without limitation
*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
*    and/or sell copies of the Software, and to permit persons to whom the
*    Software is furnished to do so, subject to the following conditions:
*
*    The above copyright notice and this permission notice shall be included in
*    all copies or substantial portions of the Software.
*
*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
*    DEALINGS IN THE SOFTWARE.
*
*****************************************************************************/

/* WARNING! AUTO-GENERATED, DO NOT MODIFY MANUALLY */

#include <stdlib.h>
#include <string.h>
#include "vsi_nn_prv.h"
#include "utils/vsi_nn_util.h"
#include "kernel/vsi_nn_kernel.h"
#include "libnnext/vsi_nn_libnnext_resource.h"


static const char a_times_b_plus_c_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniA_Times_B_2x8;\n\
_viv_uniform VXC_512Bits uniA_Plus_B_2x8;\n\
__kernel void a_times_b_plus_c_F16_F16_F16toF16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_half8   src0, src1, src2, dst;\n\
    vxc_ushort8 vec0, vec1, vec2, result;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage2DArray(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void a_times_b_plus_c_F16_F16_F16toF16_2D\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_half8   src0, src1, src2, dst;\n\
    vxc_ushort8 vec0, vec1, vec2, result;\n\
\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniA_Times_B_lo_4x4;\n\
_viv_uniform VXC_512Bits uniA_Times_B_hi_4x4;\n\
__kernel void a_times_b_plus_c_F16_F16_F32toF16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_half8   src0, src1, dst;\n\
    vxc_ushort8 vec0, vec1, result;\n\
    float4 b0, b1;\n\
    float4 dst0, dst1;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    b0 = read_imagef(input2, coord);\n\
    coord.x += 4;\n\
    b1 = read_imagef(input2, coord);\n\
    coord.x -= 4;\n\
\n\
    VXC_DP4x4(dst0, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_lo_4x4);\n\
    VXC_DP4x4(dst1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_hi_4x4);\n\
    dst0 += b0;\n\
    dst1 += b1;\n\
\n\
    half4 t0, t1;\n\
    _viv_asm(CONV, t0, dst0);\n\
    _viv_asm(CONV, t1, dst1);\n\
    VXC_DP2x8(dst, t0, t1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void a_times_b_plus_c_F16_F16_F32toF16_2D\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_t         input2,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    vxc_half8   src0, src1, dst;\n\
    vxc_ushort8 vec0, vec1, result;\n\
    float4 b0, b1;\n\
    float4 dst0, dst1;\n\
\n\
    VXC_ReadImage(vec0, input0, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    b0 = read_imagef(input2, coord.xy);\n\
    coord.z = coord.x + 4;\n\
    b1 = read_imagef(input2, coord.zy);\n\
\n\
    VXC_DP4x4(dst0, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_lo_4x4);\n\
    VXC_DP4x4(dst1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniA_Times_B_hi_4x4);\n\
    dst0 += b0;\n\
    dst1 += b1;\n\
\n\
    half4 t0, t1;\n\
    _viv_asm(CONV, t0, dst0);\n\
    _viv_asm(CONV, t1, dst1);\n\
    VXC_DP2x8(dst, t0, t1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
    _viv_asm(COPY, result, dst, 16);\n\
\n\
    VXC_WriteImage(output, coord, result, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of a_times_b_plus_c_vx*/

static const char add_mean_std_norm_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************Tensor add mean stddev norm float16*********************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform float rsEps;\n\
_viv_uniform VXC_512Bits uniAddFp16_2x8;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniAddFp16toFp32Lo_4x4;\n\
_viv_uniform VXC_512Bits uniAddFp16toFp32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
// one group(16 threads) calculates one row\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_F16_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    float eps)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0, src1, src2;\n\
    float pSum = 0, pSqr = 0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_half8 in_h, in_h1, in_h2;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, in_h1, src1, 16);\n\
        VXC_DP2x8(in_h2, in_h, in_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16_2x8);\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, in_h2, in_h2, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        pSum += sumsqr.x;\n\
        pSqr += sumsqr.y;\n\
    }\n\
\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, in_h1, src1, 16);\n\
\n\
        vxc_float4 in_f0, in_f1;\n\
        VXC_DP4x4(in_f0, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
               uniAddFp16toFp32Lo_4x4);\n\
        VXC_DP4x4(in_f1, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16toFp32Hi_4x4);\n\
\n\
        vxc_float4 norm0, norm1;\n\
        half4 norm_h0, norm_h1;\n\
\n\
        norm0 = in_f0 * stddev_inv + rMeanStd;\n\
        norm1 = in_f1 * stddev_inv + rMeanStd;\n\
        _viv_asm(CONV, norm_h0, norm0);\n\
        _viv_asm(CONV, norm_h1, norm1);\n\
\n\
        VXC_DP2x8(src2, norm_h0, norm_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_1_Lo_2x8;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_U8_U8toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    float eps)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_uchar8 src0, src1;\n\
    vxc_short8 src2;\n\
    float pSum = 0, pSqr = 0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_half8 in_h, in_h1, in_h2;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    vxc_ushort8 ms0, ms1;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, ms1, multAndoutZP1, 16);\n\
\n\
    for(; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(in_h, src0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
        VXC_DP2x8(in_h1, src1, ms1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_1_Lo_2x8);\n\
        VXC_DP2x8(in_h2, in_h, in_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16_2x8);\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, in_h2, in_h2, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        pSum += sumsqr.x;\n\
        pSqr += sumsqr.y;\n\
    }\n\
\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(in_h, src0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
        VXC_DP2x8(in_h1, src1, ms1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_1_Lo_2x8);\n\
\n\
        vxc_float4 in_f0, in_f1;\n\
        VXC_DP4x4(in_f0, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
               uniAddFp16toFp32Lo_4x4);\n\
        VXC_DP4x4(in_f1, in_h, in_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniAddFp16toFp32Hi_4x4);\n\
\n\
        vxc_float4 norm0, norm1;\n\
        half4 norm_h0, norm_h1;\n\
\n\
        norm0 = in_f0 * stddev_inv + rMeanStd;\n\
        norm1 = in_f1 * stddev_inv + rMeanStd;\n\
        _viv_asm(CONV, norm_h0, norm0);\n\
        _viv_asm(CONV, norm_h1, norm1);\n\
\n\
        VXC_DP2x8(src2, norm_h0, norm_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16ScaleToFp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16ScaleToFp32Sec_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Secd_4x4;\n\
_viv_uniform float inScale_i16;\n\
_viv_uniform float inScale1_i16;\n\
\n\
// one group(16 threads) calculates one row\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_I16_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t input1,\n\
    image2d_array_t output,\n\
    float eps)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0, src1, src2;\n\
    float pSum = 0, pSqr = 0;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    half scale_h, scale_h1;\n\
    _viv_asm(CONV, scale_h, inScale_i16);\n\
    _viv_asm(CONV, scale_h1, inScale1_i16);\n\
    float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3;\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
\n\
    for(; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(tmpVal0, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal1, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        VXC_DP4x4(tmpVal2, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal3, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        tmpVal0 += tmpVal2;\n\
        tmpVal1 += tmpVal3;\n\
\n\
        vxc_float4 sumsqr;\n\
        sumsqr = tmpVal0 * tmpVal0 + tmpVal1 * tmpVal1; // sqr\n\
        tmpVal2 = tmpVal0 + tmpVal1; // pre sum\n\
\n\
        pSum += dot(tmpVal2, one);\n\
        pSqr += dot(sumsqr, one);\n\
    }\n\
\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 data0;\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(tmpVal0, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal1, src0, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        VXC_DP4x4(tmpVal2, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Fst_4x4);\n\
        VXC_DP4x4(tmpVal3, src1, scale_h1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertInt16ScaleToFp32Sec_4x4);\n\
        tmpVal0 += tmpVal2;\n\
        tmpVal1 += tmpVal3;\n\
\n\
        vxc_float4 norm0, norm1;\n\
        half4 norm_h0, norm_h1;\n\
\n\
        norm0 = tmpVal0 * stddev_inv + rMeanStd;\n\
        norm1 = tmpVal1 * stddev_inv + rMeanStd;\n\
        _viv_asm(CONV, norm_h0, norm0);\n\
        _viv_asm(CONV, norm_h1, norm1);\n\
\n\
        VXC_DP2x8(src2, norm_h0, norm_h1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of add_mean_std_norm_vx*/

static const char argmax_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int inputWidth;\n\
_viv_uniform VXC_512Bits uniPackedIdxAddSat_2x8;\n\
_viv_uniform VXC_512Bits uniSrcT2DstT_2x8;\n\
\n\
#define TENSOR_ARGMAX_AXIS0_8BITS(src_type_name, dst_type_name, src_type, \\\n\
                                cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmax_axis0_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type maxValue, maxVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage2DArray(maxVec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(value, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value > maxVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        maxVec = max(maxVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMax3_Integer(maxValue, maxVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMax3_Integer(maxValue, maxValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    maxValue.s01234567 = maxValue.s00000000; \\\n\
 \\\n\
    cond_type1 _maxVal; \\\n\
    VXC_Clamp(_maxVal, maxVec, maxValue, maxValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _maxVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _maxVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS0_8BITS(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMAX_AXIS0_8BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmax_axis0_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    src_type maxValue, maxVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage(maxVec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(value, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value > maxVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        maxVec = max(maxVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMax3_Integer(maxValue, maxVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMax3_Integer(maxValue, maxValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    maxValue.s01234567 = maxValue.s00000000; \\\n\
 \\\n\
    cond_type1 _maxVal; \\\n\
    VXC_Clamp(_maxVal, maxVec, maxValue, maxValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _maxVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _maxVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_8BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalf2Float32_4x4;\n\
\n\
#define TENSOR_ARGMAX_AXIS0_F16_2D(dst_type_name, dst_type) \\\n\
__kernel void argmax_axis0_F16to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 maxValue, value; \\\n\
 \\\n\
    VXC_ReadImage(vec0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(maxValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(vec1, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value > maxValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        maxValue = value > maxValue ? value : maxValue; \\\n\
    } \\\n\
 \\\n\
    float4 maxVec; \\\n\
    float2 maxVal2 = maxValue.xy > maxValue.zw ? maxValue.xy : maxValue.zw; \\\n\
    maxVec.x = maxVal2.x > maxVal2.y ? maxVal2.x : maxVal2.y; \\\n\
    int4 condition; \\\n\
    condition = maxVec.xxxx == maxValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS0_F16_2D(I16, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_F16_2D(U8,  vxc_uchar8)\n\
\n\
\n\
#define TENSOR_ARGMAX_AXIS0_F16(dst_type_name, dst_type) \\\n\
__kernel void argmax_axis0_F16to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 maxValue, value; \\\n\
 \\\n\
    VXC_ReadImage2DArray(vec0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(maxValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(vec1, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value > maxValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        maxValue = value > maxValue ? value : maxValue; \\\n\
    } \\\n\
 \\\n\
    float4 maxVec; \\\n\
    float2 maxVal2 = maxValue.xy > maxValue.zw ? maxValue.xy : maxValue.zw; \\\n\
    maxVec.x = maxVal2.x > maxVal2.y ? maxVal2.x : maxVal2.y; \\\n\
    int4 condition; \\\n\
    condition = maxVec.xxxx == maxValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS0_F16(I16, vxc_ushort8)\n\
TENSOR_ARGMAX_AXIS0_F16(U8,  vxc_uchar8)\n\
"; /* end of argmax_axis0_vx*/

static const char argmax_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
\n\
#define TENSOR_ARGMAX_AXIS1_16BITS(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmax_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    copy_type max; \\\n\
    VXC_ReadImage2DArray(max, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, maxVal, max, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMax3_##inst_type(maxVal, maxVal, maxVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, max, maxVal, 16); \\\n\
       axis = (max == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS1_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMAX_AXIS1_16BITS_2D(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
__kernel void argmax_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    copy_type max; \\\n\
    VXC_ReadImage(max, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, maxVal, max, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMax3_##inst_type(maxVal, maxVal, maxVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, max, maxVal, 16); \\\n\
       axis = (max == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS1_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMAX_AXIS1_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmax_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage2DArray(maxVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_8BITS(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMAX_AXIS1_8BITS(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
\n\
#define TENSOR_ARGMAX_AXIS1_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmax_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage(maxVal, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(src, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMAX_AXIS1_8BITS_2D(U8,  U8,  vxc_uchar16, vxc_uchar16)"; /* end of argmax_axis1_vx*/

static const char argmax_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
\n\
#define TENSOR_ARGMAX_AXIS2_16BITS(src_type_name, dst_type_name,\\\n\
                src_type, copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    copy_type max; \\\n\
    VXC_ReadImage2DArray(max, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, maxVal, max, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       VXC_VertMax3_##inst_type(maxVal, maxVal, maxVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, max, maxVal, 16); \\\n\
       axis = (max == vec) ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS2_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMAX_AXIS2_16BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMAX_AXIS2_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
\n\
#define TENSOR_ARGMAX_AXIS2_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    src_type src; \\\n\
    src_type maxVal; \\\n\
    VXC_ReadImage2DArray(maxVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       maxVal = max(maxVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, maxVal, maxVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_8BITS(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_8BITS(I8,  U8,  vxc_char8,  vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS2_8BITS(U8,  I16, vxc_uchar8, vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_8BITS(U8,  U8,  vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMAX_AXIS2_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmax_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(I8,  U8,  vxc_char8,  vxc_uchar8)\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(U8,  I16, vxc_uchar8, vxc_short8)\n\
TENSOR_ARGMAX_AXIS2_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_uchar8)\n\
"; /* end of argmax_axis2_vx*/

static const char argmin_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int inputWidth;\n\
_viv_uniform VXC_512Bits uniPackedIdxAddSat_2x8;\n\
_viv_uniform VXC_512Bits uniSrcT2DstT_2x8;\n\
\n\
#define TENSOR_ARGMIN_AXIS0_8BITS(src_type_name, dst_type_name, src_type, \\\n\
                                cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmin_axis0_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type minValue, minVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage2DArray(minVec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(value, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value < minVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        minVec = min(minVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMin3_Integer(minValue, minVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(minValue, minValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    minValue.s01234567 = minValue.s00000000; \\\n\
 \\\n\
    cond_type1 _minVal; \\\n\
    VXC_Clamp(_minVal, minVec, minValue, minValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _minVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _minVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS0_8BITS(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMIN_AXIS0_8BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    cond_type0, cond_type1, dst_type, cond_type) \\\n\
__kernel void argmin_axis0_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    src_type minValue, minVec, value; \\\n\
    dst_type packIdx, currIdx; \\\n\
 \\\n\
    VXC_ReadImage(minVec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 8; \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(value, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
 \\\n\
        currIdx = currIdx + 8; \\\n\
        dst_type condition; \\\n\
        cond_type0 src_condition0 = value < minVec; \\\n\
        cond_type1 src_condition; \\\n\
        _viv_asm(COPY, src_condition, src_condition0, 8); \\\n\
        cond_type condition_tmp; \\\n\
        VXC_DP2x8(condition_tmp, src_condition, src_condition, \\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniSrcT2DstT_2x8); \\\n\
        _viv_asm(COPY, condition, condition_tmp, 16); \\\n\
        packIdx = condition ? currIdx : packIdx; \\\n\
        minVec = min(minVec, value); \\\n\
    } \\\n\
 \\\n\
    VXC_HorzMin3_Integer(minValue, minVec, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(minValue, minValue.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    minValue.s01234567 = minValue.s00000000; \\\n\
 \\\n\
    cond_type1 _minVal; \\\n\
    VXC_Clamp(_minVal, minVec, minValue, minValue, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    _minVal += 1; \\\n\
 \\\n\
    VXC_DP2x8(packIdx, packIdx, _minVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniPackedIdxAddSat_2x8); \\\n\
 \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_HorzMin3_Integer(packIdx, packIdx.s035, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.yz, packIdx, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I8,  I16, vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I8,  U8,  vxc_char8,  vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(U8,  I16, vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_char8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_ushort8, vxc_short8, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_8BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_ushort8, vxc_uchar8, vxc_uchar8)\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalf2Float32_4x4;\n\
\n\
#define TENSOR_ARGMIN_AXIS0_F16_2D(dst_type_name, dst_type) \\\n\
__kernel void argmin_axis0_F16to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), 0, 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 minValue, value; \\\n\
 \\\n\
    VXC_ReadImage(vec0, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(minValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage(vec1, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value < minValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        minValue = value < minValue ? value : minValue; \\\n\
    } \\\n\
 \\\n\
    float4 minVec; \\\n\
    float2 minVal2 = minValue.xy < minValue.zw ? minValue.xy : minValue.zw; \\\n\
    minVec.x = minVal2.x < minVal2.y ? minVal2.x : minVal2.y; \\\n\
    int4 condition; \\\n\
    condition = minVec.xxxx == minValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS0_F16_2D(I16, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_F16_2D(U8,  vxc_uchar8)\n\
\n\
\n\
#define TENSOR_ARGMIN_AXIS0_F16(dst_type_name, dst_type) \\\n\
__kernel void argmin_axis0_F16to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    vxc_short8 vec0, vec1; \\\n\
    vxc_half8 src; \\\n\
    uint4 packIdx, currIdx; \\\n\
    float4 minValue, value; \\\n\
 \\\n\
    VXC_ReadImage2DArray(vec0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec0, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, currIdx, packedArgIdx, 16); \\\n\
    coord.x += 4; \\\n\
    VXC_DP4x4(minValue, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
    for (; coord.x < inputWidth; ) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(vec1, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, src, vec1, 16); \\\n\
        coord.x += 4; \\\n\
 \\\n\
       currIdx = currIdx + 4; \\\n\
        VXC_DP4x4(value, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertHalf2Float32_4x4); \\\n\
 \\\n\
        int4 condition; \\\n\
        condition = value < minValue; \\\n\
 \\\n\
        uint4 iCondition; \\\n\
        _viv_asm(COPY, iCondition, condition, 16); \\\n\
        packIdx = iCondition ? currIdx : packIdx; \\\n\
        minValue = value < minValue ? value : minValue; \\\n\
    } \\\n\
 \\\n\
    float4 minVec; \\\n\
    float2 minVal2 = minValue.xy < minValue.zw ? minValue.xy : minValue.zw; \\\n\
    minVec.x = minVal2.x < minVal2.y ? minVal2.x : minVal2.y; \\\n\
    int4 condition; \\\n\
    condition = minVec.xxxx == minValue; \\\n\
    uint4 iCondition; \\\n\
    _viv_asm(COPY, iCondition, condition, 16); \\\n\
    iCondition += 1; \\\n\
 \\\n\
    packIdx = mad_sat(iCondition, 0xFFFFFFFF, packIdx); \\\n\
 \\\n\
    uint2 val2 = packIdx.xy < packIdx.zw ? packIdx.xy : packIdx.zw; \\\n\
    val2.x = val2.x < val2.y ? val2.x : val2.y; \\\n\
 \\\n\
    dst_type dst; \\\n\
    _viv_asm(COPY, dst, val2, 4); \\\n\
    VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS0_F16(I16, vxc_ushort8)\n\
TENSOR_ARGMIN_AXIS0_F16(U8,  vxc_uchar8)\n\
"; /* end of argmin_axis0_vx*/

static const char argmin_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
\n\
#define TENSOR_ARGMIN_AXIS1_16BITS(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    copy_type min; \\\n\
    VXC_ReadImage2DArray(min, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, minVal, min, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMin3_##inst_type(minVal, minVal, minVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, min, minVal, 16); \\\n\
       axis = (min == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS1_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMIN_AXIS1_16BITS_2D(src_type_name, dst_type_name, src_type,\\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    copy_type min; \\\n\
    VXC_ReadImage(min, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, minVal, min, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       VXC_VertMin3_##inst_type(minVal, minVal, minVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, min, minVal, 16); \\\n\
       axis = (min == vec) ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xz, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS1_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMIN_AXIS1_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmin_axis1_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, get_global_id(1), 0); \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    VXC_ReadImage2DArray(minVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       minVal = min(minVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, minVal, minVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_8BITS(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMIN_AXIS1_8BITS(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
\n\
#define TENSOR_ARGMIN_AXIS1_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
__kernel void argmin_axis1_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), argLenSub1, 0, 0); \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    VXC_ReadImage(minVal, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.y --; \\\n\
    for (;coord.y >= 0;) \\\n\
    { \\\n\
       VXC_ReadImage(src, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.y --; \\\n\
       packIdx --; \\\n\
       minVal = min(minVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, minVal, minVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xz, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(I8,  I16, vxc_char16,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(U8,  I16, vxc_uchar16, vxc_short8)\n\
TENSOR_ARGMIN_AXIS1_8BITS_2D(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
\n\
"; /* end of argmin_axis1_vx*/

static const char argmin_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedArgIdx;\n\
_viv_uniform int argLenSub1;\n\
_viv_uniform VXC_512Bits uniExtractData_2x8;\n\
\n\
#define TENSOR_ARGMIN_AXIS2_16BITS(src_type_name, dst_type_name,\\\n\
                src_type, copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    copy_type vec; \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    copy_type min; \\\n\
    VXC_ReadImage2DArray(min, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, minVal, min, 16); \\\n\
    axis_type axis; \\\n\
    axis_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(vec, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       _viv_asm(COPY, src, vec, 16); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       VXC_VertMin3_##inst_type(minVal, minVal, minVal, src, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
       _viv_asm(COPY, min, minVal, 16); \\\n\
       axis = (min == vec) ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    dst_type dst_axis; \\\n\
    VXC_DP2x8(dst_axis, axis, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractData_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst_axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_16BITS(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS2_16BITS(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
#define TENSOR_ARGMIN_AXIS2_16BITS_2D(src_type_name, dst_type_name, src_type, \\\n\
                                    copy_type, axis_type, dst_type, inst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8, vxc_short8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_short8, vxc_uchar8, Half)\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, Integer)\n\
TENSOR_ARGMIN_AXIS2_16BITS_2D(I16, U8,  vxc_short8, vxc_short8, vxc_short8, vxc_uchar8, Integer)\n\
\n\
\n\
#define TENSOR_ARGMIN_AXIS2_8BITS(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), argLenSub1, 0); \\\n\
    src_type src; \\\n\
    src_type minVal; \\\n\
    VXC_ReadImage2DArray(minVal, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst_type axis; \\\n\
    dst_type packIdx; \\\n\
 \\\n\
    _viv_asm(COPY, axis, packedArgIdx, 16); \\\n\
    _viv_asm(COPY, packIdx, packedArgIdx, 16); \\\n\
 \\\n\
    coord.z --; \\\n\
    do \\\n\
    { \\\n\
       VXC_ReadImage2DArray(src, input, coord.xyzw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
       coord.z --; \\\n\
       packIdx --; \\\n\
       minVal = min(minVal, src); \\\n\
       dst_type condition; \\\n\
       VXC_Clamp(condition, src, minVal, minVal, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
       axis = condition ? packIdx : axis; \\\n\
    } while (coord.z >= 0); \\\n\
 \\\n\
    VXC_WriteImage(output, coord.xy, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_8BITS(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS(I8,  U8,  vxc_char8,  vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS2_8BITS(U8,  I16, vxc_uchar8, vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS(U8,  U8,  vxc_uchar8, vxc_uchar8)\n\
\n\
#define TENSOR_ARGMIN_AXIS2_8BITS_2D(src_type_name, dst_type_name, src_type, dst_type) \\\n\
    __kernel void argmin_axis2_##src_type_name##to##dst_type_name##_2D( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
        int  axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    dst_type axis = (dst_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_WriteImage(output, coord, axis, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(I8,  I16, vxc_char8,  vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(I8,  U8,  vxc_char8,  vxc_uchar8)\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(U8,  I16, vxc_uchar8, vxc_short8)\n\
TENSOR_ARGMIN_AXIS2_8BITS_2D(U8,  U8,  vxc_uchar8, vxc_uchar8)\n\
"; /* end of argmin_axis2_vx*/

static const char batchnorm_single_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDatatoF32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoF32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define BATCH_NORM_SH_IMPL(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst1( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage2DArray(_gamma, Gamma, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst1_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage(_gamma, Gamma, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta = read_imagef(Beta, coord.xy); \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst0( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage2DArray(_gamma, Gamma, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta0 = read_imagef(Beta, coord); \\\n\
    coord.x += 4; \\\n\
    float4 beta1 = read_imagef(Beta, coord); \\\n\
    coord.x -= 4; \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F16_F32to##name1##_brdcst0_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    coord.z += 4; \\\n\
    vxc_ushort8 _mean, _var, _gamma; \\\n\
    vxc_half8 mean, var, gamma; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    VXC_ReadImage(_gamma, Gamma, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, gamma, _gamma, 16); \\\n\
    float4 beta0 = read_imagef(Beta, coord.xy); \\\n\
    float4 beta1 = read_imagef(Beta, coord.zy); \\\n\
 \\\n\
    float4 src0, src1, m, v, g; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * g + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(g, gamma, gamma, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    g = g * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * g + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
"; /* end of batchnorm_single_vx*/

static const char batchnorm_single_f32_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDatatoF32_0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoF32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
\n\
#define BATCH_NORM_SH_IMPL(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst1( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    int4 coord_in = coord; \\\n\
    int depth = get_image_array_size(Gamma); \\\n\
    _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
    float4 gamma = read_imagef(Gamma, coord_in); \\\n\
    coord_in.z = coord.z; \\\n\
    depth = get_image_array_size(Beta); \\\n\
    _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
    float4 beta = read_imagef(Beta, coord_in); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    float4 gamma0 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    float4 gamma1 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst1_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    coord.z = coord.x + 4; \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    float4 gamma = read_imagef(Gamma, coord.xy); \\\n\
    float4 beta = read_imagef(Beta, coord.xy); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    float4 gamma0 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta.xxxx; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    float4 gamma1 = gamma.xxxx * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta.xxxx; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst0( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage2DArray(vec, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage2DArray(_mean, Mean, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage2DArray(_var, Variance, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    int4 coord_in0 = coord; \\\n\
    int depth = get_image_array_size(Gamma); \\\n\
    _viv_asm(CLAMP0MAX, coord_in0.z, coord_in0.z, depth - 1); \\\n\
    float4 gamma0 = read_imagef(Gamma, coord_in0); \\\n\
    int4 coord_in1 = coord; \\\n\
    depth = get_image_array_size(Beta); \\\n\
    _viv_asm(CLAMP0MAX, coord_in1.z, coord_in1.z, depth - 1); \\\n\
    float4 beta0 = read_imagef(Beta, coord_in1); \\\n\
    coord_in0.x += 4; \\\n\
    coord_in1.x += 4; \\\n\
    float4 gamma1 = read_imagef(Gamma, coord_in0); \\\n\
    float4 beta1 = read_imagef(Beta, coord_in1); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    gamma0 = gamma0 * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    gamma1 = gamma1 * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
\n\
#define BATCH_NORM_SH_IMPL_AXIS1_2D(name0, name1, src_type, read_type, conv_type, dst_type, save_type) \\\n\
__kernel void batch_norm_##name0##_F16_F16_F32_F32to##name1##_brdcst0_2D( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       Mean, \\\n\
    __read_only  image2d_t       Variance, \\\n\
    __read_only  image2d_t       Gamma, \\\n\
    __read_only  image2d_t       Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    read_type vec; \\\n\
    src_type src; \\\n\
    VXC_ReadImage(vec, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, vec, 16); \\\n\
    coord.z += 4; \\\n\
    vxc_ushort8 _mean, _var; \\\n\
    vxc_half8 mean, var; \\\n\
    VXC_ReadImage(_mean, Mean, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, mean, _mean, 16); \\\n\
    VXC_ReadImage(_var, Variance, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, var, _var, 16); \\\n\
    float4 gamma0 = read_imagef(Gamma, coord.xy); \\\n\
    float4 gamma1 = read_imagef(Gamma, coord.zy); \\\n\
    float4 beta0 = read_imagef(Beta, coord.xy); \\\n\
    float4 beta1 = read_imagef(Beta, coord.zy); \\\n\
 \\\n\
    float4 src0, src1, m, v; \\\n\
    VXC_DP4x4(src0, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_0_4x4); \\\n\
    gamma0 = gamma0 * rsqrt(v + eps); \\\n\
    src0 = src0 * input_scale + input_tail; \\\n\
    src0 = (src0 - m) * gamma0 + beta0; \\\n\
    src0 = src0 * output_scale + output_zp; \\\n\
    VXC_DP4x4(src1, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(m, mean, mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    VXC_DP4x4(v, var, var, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDatatoF32_1_4x4); \\\n\
    gamma1 = gamma1 * rsqrt(v + eps); \\\n\
    src1 = src1 * input_scale + input_tail; \\\n\
    src1 = (src1 - m) * gamma1 + beta1; \\\n\
    src1 = src1 * output_scale + output_zp; \\\n\
 \\\n\
    conv_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, src0); \\\n\
    _viv_asm(CONV_RTE, dst1, src1); \\\n\
    dst_type tmp; \\\n\
    save_type dst; \\\n\
    VXC_DP2x8(tmp, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, tmp, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, F16, vxc_half8,   vxc_ushort8, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I16, vxc_half8,   vxc_ushort8, int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, U8,  vxc_half8,   vxc_ushort8, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(F16, I8,  vxc_half8,   vxc_ushort8, int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, I16, vxc_short8,  vxc_short8,  int4,  vxc_short8,  vxc_short8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I16, F16, vxc_short8,  vxc_short8,  half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  U8,  vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16, vxc_uchar16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(U8,  F16, vxc_uchar16, vxc_uchar16, half4, vxc_half8,   vxc_ushort8)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  I8,  vxc_char16,  vxc_char16,  int4,  vxc_char16,  vxc_char16)\n\
BATCH_NORM_SH_IMPL_AXIS1_2D(I8,  F16, vxc_char16,  vxc_char16,  half4, vxc_half8,   vxc_ushort8)\n\
"; /* end of batchnorm_single_f32_vx*/

static const char cast_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataConvert_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
#define CAST_PROCESS(read_type, src_type, dst_type, write_type, read_fun, write_fun) \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst_val; \\\n\
    write_type write_val; \\\n\
    read_fun(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    VXC_DP2x8(dst_val, src_val, src_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniDataConvert_2x8); \\\n\
    _viv_asm(COPY, write_val, dst_val, 16); \\\n\
    write_fun(output, coord, write_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define CAST_FUN(src_name, dst_name, read_type, src_type, dst_type, write_type) \\\n\
__kernel void cast_##src_name##to##dst_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    CAST_PROCESS(read_type, src_type, dst_type, write_type, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
CAST_FUN(F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
CAST_FUN(F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8)\n\
CAST_FUN(F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN(I16, F16, vxc_short8, vxc_short8, vxc_half8,  vxc_short8)\n\
CAST_FUN(I16, I8,  vxc_short8, vxc_short8, vxc_char8,  vxc_char8)\n\
CAST_FUN(I16, U8,  vxc_short8, vxc_short8, vxc_uchar8, vxc_uchar8)\n\
CAST_FUN(I8, F16, vxc_char8,   vxc_char8,  vxc_half8,  vxc_short8)\n\
CAST_FUN(I8, I16, vxc_char8,   vxc_char8,  vxc_short8, vxc_short8)\n\
CAST_FUN(I8, U8,  vxc_char8,   vxc_char8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN(U8, F16, vxc_uchar8,  vxc_uchar8, vxc_half8,  vxc_short8)\n\
CAST_FUN(U8, I16, vxc_uchar8,  vxc_uchar8, vxc_short8, vxc_short8)\n\
CAST_FUN(U8, I8,  vxc_uchar8,  vxc_uchar8, vxc_char8,  vxc_char8)\n\
\n\
\n\
#define CAST_FUN_2D(src_name, dst_name, read_type, src_type, dst_type, write_type) \\\n\
__kernel void cast_##src_name##to##dst_name##_2D( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    CAST_PROCESS(read_type, src_type, dst_type, write_type, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
CAST_FUN_2D(F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8)\n\
CAST_FUN_2D(F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8)\n\
CAST_FUN_2D(F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN_2D(I16, F16, vxc_short8, vxc_short8, vxc_half8,  vxc_short8)\n\
CAST_FUN_2D(I16, I8,  vxc_short8, vxc_short8, vxc_char8,  vxc_char8)\n\
CAST_FUN_2D(I16, U8,  vxc_short8, vxc_short8, vxc_uchar8, vxc_uchar8)\n\
CAST_FUN_2D(I8, F16, vxc_char8,   vxc_char8,  vxc_half8,  vxc_short8)\n\
CAST_FUN_2D(I8, I16, vxc_char8,   vxc_char8,  vxc_short8, vxc_short8)\n\
CAST_FUN_2D(I8, U8,  vxc_char8,   vxc_char8,  vxc_uchar8, vxc_uchar8)\n\
CAST_FUN_2D(U8, F16, vxc_uchar8,  vxc_uchar8, vxc_half8,  vxc_short8)\n\
CAST_FUN_2D(U8, I16, vxc_uchar8,  vxc_uchar8, vxc_short8, vxc_short8)\n\
CAST_FUN_2D(U8, I8,  vxc_uchar8,  vxc_uchar8, vxc_char8,  vxc_char8)\n\
\n\
#define CAST_TO_BOOL_PROCESS(src_type, tmp_type, read_fun, write_fun) \\\n\
    src_type   src_val; \\\n\
    tmp_type   tmp_val; \\\n\
    vxc_char8   dst_val; \\\n\
    read_fun(src_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    tmp_val  = (src_val != 0); \\\n\
    tmp_val *= (-1); \\\n\
    VXC_DP2x8(dst_val, tmp_val, tmp_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDataConvert_2x8); \\\n\
    write_fun(output, coord, dst_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define CAST_TO_BOOL_FUN(src_name, src_type, tmp_type) \\\n\
__kernel void cast_##src_name##toBOOL8( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    CAST_TO_BOOL_PROCESS(src_type, tmp_type, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN(F16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN(I16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN(I8,  vxc_char8,  vxc_char8)\n\
CAST_TO_BOOL_FUN(U8,  vxc_uchar8, vxc_char8)\n\
\n\
#define CAST_TO_BOOL_FUN_2D(src_name, src_type, tmp_type) \\\n\
__kernel void cast_##src_name##toBOOL8_2D( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    CAST_TO_BOOL_PROCESS(src_type, tmp_type, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN_2D(F16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN_2D(I16, vxc_short8, vxc_short8)\n\
CAST_TO_BOOL_FUN_2D(I8,  vxc_char8,  vxc_char8)\n\
CAST_TO_BOOL_FUN_2D(U8,  vxc_uchar8, vxc_char8)\n\
\n\
#define CAST_F32orI32_PROCESS(src_type, dst_type, read_fun, write_fun) \\\n\
    src_type   src_val0, src_val1; \\\n\
    dst_type   dst_val; \\\n\
    int4 tmpData1, tmpData2; \\\n\
    src_val0 = read_fun(input, coord); \\\n\
    coord.x += 4; \\\n\
    src_val1 = read_fun(input, coord); \\\n\
    tmpData1 = convert_int4(src_val0); \\\n\
    tmpData2 = convert_int4(src_val1); \\\n\
    VXC_DP2x8(dst_val, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    coord.x -= 4; \\\n\
    write_fun(output, coord, dst_val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define CAST_F32orI32_FUN(src_name, dst_name, src_type, dst_type, read_fun) \\\n\
__kernel void cast_##src_name##to##dst_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    CAST_F32orI32_PROCESS(src_type, dst_type, read_fun, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
CAST_F32orI32_FUN(F32, I16, float4, vxc_short8, read_imagef)\n\
CAST_F32orI32_FUN(F32, I8,  float4, vxc_char8,  read_imagef)\n\
CAST_F32orI32_FUN(F32, U8,  float4, vxc_uchar8, read_imagef)\n\
CAST_F32orI32_FUN(I32, I16, int4,   vxc_short8, read_imagei)\n\
CAST_F32orI32_FUN(I32, I8,  int4,   vxc_char8,  read_imagei)\n\
CAST_F32orI32_FUN(I32, U8,  int4,   vxc_uchar8, read_imagei)\n\
\n\
\n\
#define CAST_F32orI32_FUN_2D(src_name, dst_name, src_type, dst_type, read_fun) \\\n\
__kernel void cast_##src_name##to##dst_name##_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    CAST_F32orI32_PROCESS(src_type, dst_type, read_fun, VXC_WriteImage) \\\n\
}\n\
\n\
CAST_F32orI32_FUN_2D(F32, I16, float4, vxc_short8, read_imagef)\n\
CAST_F32orI32_FUN_2D(F32, I8,  float4, vxc_char8,  read_imagef)\n\
CAST_F32orI32_FUN_2D(F32, U8,  float4, vxc_uchar8, read_imagef)\n\
CAST_F32orI32_FUN_2D(I32, I16, int4,   vxc_short8, read_imagei)\n\
CAST_F32orI32_FUN_2D(I32, I8,  int4,   vxc_char8,  read_imagei)\n\
CAST_F32orI32_FUN_2D(I32, U8,  int4,   vxc_uchar8, read_imagei)\n\
\n\
"; /* end of cast_vx*/

static const char clip_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedMinData_FP16;\n\
_viv_uniform int4 packedMaxData_FP16;\n\
_viv_uniform VXC_512Bits uniConvertF16toInt_2x8;\n\
_viv_uniform int2 multAndoutZP;\n\
_viv_uniform VXC_512Bits uniDataMulAndPostShift_2x8;\n\
\n\
#define TENSORCLIP_F16TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 vec0, dst; \\\n\
    vxc_half8  src0, src1, minHf, maxHf; \\\n\
    read_fun(vec0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vec0, 16); \\\n\
    _viv_asm(COPY, minHf, packedMinData_FP16, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData_FP16, 16); \\\n\
    VXC_Clamp_Half(src1, src0, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src1, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_F16toF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_F16toF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_F16TOINT_PROCESS(read_fun, write_fun, dst_type) \\\n\
    vxc_short8 vec0; \\\n\
    dst_type dst; \\\n\
    vxc_half8  src0, src1, minHf, maxHf; \\\n\
    read_fun(vec0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vec0, 16); \\\n\
    _viv_asm(COPY, minHf, packedMinData_FP16, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData_FP16, 16); \\\n\
    VXC_Clamp_Half(src1, src0, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    VXC_DP2x8(dst, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertF16toInt_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_F16toI16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, vxc_short8)\n\
}\n\
\n\
__kernel void clip_F16toI16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage, VXC_WriteImage, vxc_short8)\n\
}\n\
\n\
__kernel void clip_F16toI8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, vxc_char16)\n\
}\n\
\n\
__kernel void clip_F16toI8_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOINT_PROCESS(VXC_ReadImage, VXC_WriteImage, vxc_char16)\n\
}\n\
\n\
#define TENSORCLIP_F16TOU8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 vec0; \\\n\
    vxc_uchar16 dst; \\\n\
    vxc_half8  src0, src1, minHf, maxHf; \\\n\
    read_fun(vec0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vec0, 16); \\\n\
    _viv_asm(COPY, minHf, packedMinData_FP16, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData_FP16, 16); \\\n\
    VXC_Clamp_Half(src1, src0, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src1, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniDataMulAndPostShift_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_F16toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_F16TOU8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_F16toU8_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_F16TOU8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of clip_F16_vx*/

static const char clip_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntegerLo_2x8;\n\
_viv_uniform int4 packedMinData;\n\
_viv_uniform int4 packedMaxData;\n\
\n\
#define TENSORCLIP_I16TOI16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, min, max; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    _viv_asm(COPY, min, packedMinData, 16); \\\n\
    _viv_asm(COPY, max, packedMaxData, 16); \\\n\
    VXC_Clamp(src0, src0, min, max, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    write_fun(output, coord, src0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_I16toI16(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I16TOI16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I16toI16_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I16TOI16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_I16TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, dst; \\\n\
    vxc_half8  src1, src2, minHf, maxHf; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src1, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    _viv_asm(COPY, minHf, packedMinData, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData, 16); \\\n\
    VXC_Clamp_Half(src2, src1, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst, src2, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void clip_I16toF16(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I16TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I16toF16_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I16TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of clip_I16_vx*/

static const char clip_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntegerLo_2x8;\n\
_viv_uniform VXC_512Bits uniConvertIntegerHi_2x8;\n\
_viv_uniform int4 packedMinData;\n\
_viv_uniform int4 packedMaxData;\n\
\n\
#define TENSORCLIP_I8TOI8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 src0, min, max; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerHi_2x8); \\\n\
    _viv_asm(COPY, min, packedMinData, 16); \\\n\
    _viv_asm(COPY, max, packedMaxData, 16); \\\n\
    VXC_Clamp(src0, src0, min, max, VXC_MODIFIER_CLAMP(0, 15, 0, 0)); \\\n\
    write_fun(output, coord, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_I8toI8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I8TOI8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I8toI8_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I8TOI8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_I8TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 src0; \\\n\
    vxc_short8 dst0, dst1; \\\n\
    vxc_half8  src1, src2, src3, src4, minHf, maxHf; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src1, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerLo_2x8); \\\n\
    VXC_DP2x8(src2, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertIntegerHi_2x8); \\\n\
    _viv_asm(COPY, minHf, packedMinData, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData, 16); \\\n\
    VXC_Clamp_Half(src3, src1, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    VXC_Clamp_Half(src4, src2, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst0, src3, 16); \\\n\
    _viv_asm(COPY, dst1, src4, 16); \\\n\
    write_fun(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    write_fun(output, coord, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void clip_I8toF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_I8TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_I8toF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_I8TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of clip_I8_vx*/

static const char clip_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int4 packedMinData;\n\
_viv_uniform int4 packedMaxData;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define TENSORCLIP_U8TOU8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 vec0, min, max, dst; \\\n\
    read_fun(vec0, input,  coord,\\\n\
         VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, vec0, multiplier,\\\n\
         VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(dst, vec0, multiplier,\\\n\
         VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Hi_2x8); \\\n\
    _viv_asm(COPY, min, packedMinData, 16); \\\n\
    _viv_asm(COPY, max, packedMaxData, 16); \\\n\
    VXC_Clamp(dst, dst, min, max, VXC_MODIFIER_CLAMP(0, 15, 0, 0)); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void clip_U8toU8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_U8TOU8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_U8toU8_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_U8TOU8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define TENSORCLIP_U8TOF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 vec0; \\\n\
    vxc_short8 dst0, dst1; \\\n\
    vxc_half8  src1, src2, src3, src4, minHf, maxHf; \\\n\
    read_fun(vec0, input,  coord,\\\n\
         VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(src1, vec0, multiplier,\\\n\
         VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(src2, vec0, multiplier,\\\n\
         VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Hi_2x8); \\\n\
    _viv_asm(COPY, minHf, packedMinData, 16); \\\n\
    _viv_asm(COPY, maxHf, packedMaxData, 16); \\\n\
    VXC_Clamp_Half(src3, src1, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    VXC_Clamp_Half(src4, src2, minHf, maxHf, VXC_MODIFIER_CLAMP(0, 7, 0, 0)); \\\n\
    _viv_asm(COPY, dst0, src3, 16); \\\n\
    _viv_asm(COPY, dst1, src4, 16); \\\n\
    write_fun(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    write_fun(output, coord, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void clip_U8toF16(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    TENSORCLIP_U8TOF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void clip_U8toF16_2D(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                           float   minData,\n\
                           float   maxData)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    TENSORCLIP_U8TOF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of clip_U8_vx*/

static const char conv1d_ovxlib_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConv1DK3_Lo0_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Lo1_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Lo2_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Hi0_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Hi1_4x4;\n\
_viv_uniform VXC_512Bits uniConv1DK3_Hi2_4x4;\n\
_viv_uniform VXC_512Bits uniDataConvK3_2x8;\n\
_viv_uniform VXC_512Bits uniSumOrderUchar_2x8;\n\
\n\
_viv_uniform int input_ZP;\n\
_viv_uniform int weight_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float scaleOut;\n\
_viv_uniform int input_height;\n\
\n\
__kernel void conv1d_U8U8I32toU8_K3_S1(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int4 coord   = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 weight_val_s =(short)input_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
    weight_val_s.s5 = (short)weight_ZP;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(weight_val_s, weight_val, weight_val_s, \\\n\
                             VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0), uniDataConvK3_2x8);\n\
\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo0_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi0_4x4);\n\
        sum1 += dst;\n\
        coord.x += dilation;\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo1_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi1_4x4);\n\
        sum1 += dst;\n\
        coord.x += dilation;\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo2_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi2_4x4);\n\
        sum1 += dst;\n\
        coord_w.y++;\n\
        coord.z++;\n\
        coord.x = get_global_id(0);\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    VXC_WriteImage(output, coord.xy, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void conv1d_U8U8I32toU8_K3_S1_D2_D4(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int4 coord   = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 weight_val_s =(short)input_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
    weight_val_s.s5 = (short)weight_ZP;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(weight_val_s, weight_val, weight_val_s, \\\n\
                             VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0), uniDataConvK3_2x8);\n\
\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo0_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi0_4x4);\n\
        sum1 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo1_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi1_4x4);\n\
        sum1 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Lo2_4x4);\n\
        sum0 += dst;\n\
        VXC_DP4x4(dst, input_val, weight_val_s, \\\n\
                         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConv1DK3_Hi2_4x4);\n\
        sum1 += dst;\n\
        coord_w.y++;\n\
        coord.z++;\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    VXC_WriteImage(output, coord.xy, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of conv1d_ovxlib_vx*/

static const char conv1d_ovxlib_k1024_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part0_8x2;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part1_8x2;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part2_8x2;\n\
_viv_uniform VXC_512Bits uniU8Conv1d_part3_8x2;\n\
_viv_uniform VXC_512Bits uniSumOrderUchar_2x8;\n\
\n\
_viv_uniform int kernel_cnt_x16;\n\
_viv_uniform int weight_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float scaleOut;\n\
_viv_uniform int input_height;\n\
_viv_uniform int input_width;\n\
_viv_uniform int output_width;\n\
\n\
__kernel void conv1d_U8U8I32toU8_K1024_SMALL(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int  start_x = get_global_id(0) - pad_front;\n\
    int4 coord   = (int4)(start_x, get_global_id(1), 0, get_global_id(0));\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 coef;\n\
    vxc_short8 w_zp = (short)weight_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i, j;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        for (j = 0; j < kernel_cnt_x16; j++)\n\
        {\n\
            VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            VXC_ReadImage(input_val, input, coord.xz, VXC_5BITOFFSET_XY(8, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            coord_w.x += 16;\n\
            coord.x += 16;\n\
        }\n\
        coord_w.x = 0;\n\
        coord_w.y++;\n\
        coord.z++;\n\
        coord.x = start_x;\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    VXC_WriteImage(output, coord.wy, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void conv1d_U8U8I32toU8_K1024_LARGE(\n\
     __read_only image2d_array_t   input,\n\
     __read_only image2d_array_t   weight,\n\
     __read_only image2d_t         bias,\n\
    __write_only image2d_array_t   output,\n\
                 int               stride,\n\
                 int               pad_front,\n\
                 int               pad_end,\n\
                 int               dilation,\n\
                 int               overflow_policy)\n\
{\n\
    int  start_x = get_global_id(0);\n\
    int  w_left  = output_width - start_x;\n\
    int  out_x   = w_left < 8 ? get_global_id(0) - w_left : get_global_id(0);\n\
    int4 coord   = (int4)(start_x, get_global_id(1), 0, out_x);\n\
    int4 coord_w = (int4)(0, 0, get_global_id(1), 0);\n\
    float4 sum0, sum1, dst;\n\
    vxc_short8 coef;\n\
    vxc_short8 w_zp = (short)weight_ZP;\n\
    vxc_uchar16 input_val = 0, weight_val = 0;\n\
    int temp = 0, i, j;\n\
    Tensor src_tensor = create_image_from_image2d(input, 1);\n\
    uchar *src_ptr_base = (uchar *)src_image.ptr;\n\
    uchar *src_ptr;\n\
    Tensor dst_tensor = create_image_from_image2d(output, 1);\n\
    uchar *dst_ptr = (uchar *)dst_tensor.ptr;\n\
\n\
    temp = read_imagei(bias, coord.yz).x;\n\
    sum0 = convert_float(temp);\n\
    sum1 = sum0;\n\
\n\
    for (i = 0; i < input_height; i++)\n\
    {\n\
        src_ptr = src_ptr_base + (coord.x + coord.z * src_image.stride_y);\n\
        for (j = 0; j < kernel_cnt_x16; j++)\n\
        {\n\
            VXC_ReadImage2DArray(weight_val, weight, coord_w, VXC_5BITOFFSET_XY(0, 0), \\\n\
                                 VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            VXC_Vload16(input_val, src_ptr, 0);\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            src_ptr += 8;\n\
            VXC_Vload16(input_val, src_ptr, 0);\n\
            VXC_DP2x8(coef, weight_val, w_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part0_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part1_8x2);\n\
            sum0 += dst;\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part2_8x2);\n\
            VXC_DP8x2(dst, input_val, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8Conv1d_part3_8x2);\n\
            sum1 += dst;\n\
            coord_w.x += 16;\n\
            coord.x += 16;\n\
            src_ptr += 8;\n\
        }\n\
        coord_w.x = 0;\n\
        coord_w.y++;\n\
        coord.z++;\n\
        coord.x = start_x;\n\
    }\n\
\n\
    sum0 = sum0 * scaleOut + output_ZP;\n\
    sum1 = sum1 * scaleOut + output_ZP;\n\
    uchar4 result0, result1;\n\
    _viv_asm(CONV_SAT_RTE, result0, sum0);\n\
    _viv_asm(CONV_SAT_RTE, result1, sum1);\n\
    vxc_uchar8 result;\n\
    VXC_DP2x8(result, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniSumOrderUchar_2x8);\n\
    dst_ptr = dst_ptr + (coord.w + coord.y * dst_tensor.stride_y);\n\
    VXC_Vstore8(dst_ptr, 0, result);\n\
}\n\
\n\
"; /* end of conv1d_ovxlib_k1024_vx*/

static const char depth2space_crd_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_ExLo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_ExHi_2x8;\n\
_viv_uniform VXC_512Bits uniDepth2SpaceF16Blk2_lo_2x8;\n\
_viv_uniform VXC_512Bits uniDepth2SpaceF16Blk2_hi_2x8;\n\
\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_QINT(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##to##src1_type_name( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int block_e2 = block_size * block_size; \\\n\
    ushort blk = (ushort)block_size; \\\n\
    int inx = (int)((ushort)gidx / blk); \\\n\
    int iny = (int)((ushort)gidy / blk); \\\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2; \\\n\
    int4 coord_in = (int4)(inx, iny, inz, 0); \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(dst,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_QINT(U8, U8, vxc_uchar16, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_QINT(I8, I8, vxc_char16, vxc_char16)\n\
DEPTH2SPACE_CRD_QINT_TO_QINT(I16, I16, vxc_short8, vxc_short8)\n\
\n\
__kernel void depth2space_crd_F16toF16(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0);\n\
    int block_e2 = block_size * block_size;\n\
    ushort blk = (ushort)block_size;\n\
    int inx = (int)((ushort)gidx / blk);\n\
    int iny = (int)((ushort)gidy / blk);\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2;\n\
    int4 coord_in = (int4)(inx, iny, inz, 0);\n\
    vxc_short8 data;\n\
    VXC_ReadImage2DArray(data,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, coord_out, data, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_F16(src0_type_name, read_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##toF16( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int block_e2 = block_size * block_size; \\\n\
    ushort blk = (ushort)block_size; \\\n\
    int inx = (int)((ushort)gidx / blk); \\\n\
    int iny = (int)((ushort)gidy / blk); \\\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2; \\\n\
    int4 coord_in = (int4)(inx, iny, inz, 0); \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  tmpDst; \\\n\
    vxc_short8  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(tmpDst,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst, tmpDst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_F16(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_F16(I8, vxc_char16)\n\
DEPTH2SPACE_CRD_QINT_TO_F16(I16, vxc_short8)\n\
\n\
#define DEPTH2SPACE_CRD_F16_TO_QINT(src1_type_name, write_type) \\\n\
__kernel void depth2space_crd_F16to##src1_type_name( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(gidx, gidy, gidz, 0); \\\n\
    int block_e2 = block_size * block_size; \\\n\
    ushort blk = (ushort)block_size; \\\n\
    int inx = (int)((ushort)gidx / blk); \\\n\
    int iny = (int)((ushort)gidy / blk); \\\n\
    int inz = (gidx  % block_size) + (gidy % block_size) * block_size + gidz * block_e2; \\\n\
    int4 coord_in = (int4)(inx, iny, inz, 0); \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage2DArray(src,input,coord_in,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,0,0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_half8 data; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_F16_TO_QINT(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_F16_TO_QINT(I8, vxc_char16)\n\
DEPTH2SPACE_CRD_F16_TO_QINT(I16, vxc_short8)\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_QINT_BLK2(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##to##src1_type_name##_blk2( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz); \\\n\
    int4 coord_in = coord_out >> 1; \\\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4; \\\n\
    coord_in.w = coord_in.z + 1; \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyzz, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyww, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExLo_2x8); \\\n\
    VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExHi_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_QINT_BLK2(U8, U8, vxc_uchar16, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_QINT_BLK2(I8, I8, vxc_char16, vxc_char16)\n\
\n\
__kernel void depth2space_crd_F16toF16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 data0, data1, dst0, dst1;\n\
    VXC_ReadImage2DArray(data0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(data1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(dst0, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(dst1, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void depth2space_crd_I16toI16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 src0, src1, data0, data1, dst0, dst1;\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(dst0, data0, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    VXC_DP2x8(dst1, data1, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define DEPTH2SPACE_CRD_QINT_TO_F16_BLK2(src0_type_name, read_type) \\\n\
__kernel void depth2space_crd_##src0_type_name##toF16_blk2( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz); \\\n\
    int4 coord_in = coord_out >> 1; \\\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4; \\\n\
    coord_in.w = coord_in.z + 1; \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyzz, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src, input, coord_in.xyww, \\\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  tmpDst0, tmpDst1; \\\n\
    vxc_short8  dst0, dst1; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(tmpDst0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExLo_2x8); \\\n\
    VXC_DP2x8(tmpDst1, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_ExHi_2x8); \\\n\
    _viv_asm(COPY, dst0, tmpDst0, 16); \\\n\
    _viv_asm(COPY, dst1, tmpDst1, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_out.x+=8; \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_QINT_TO_F16_BLK2(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_QINT_TO_F16_BLK2(I8, vxc_char16)\n\
\n\
__kernel void depth2space_crd_I16toF16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 src0, src1, data0, data1, dst0, dst1;\n\
    vxc_half8 tmpDst0, tmpDst1;\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(tmpDst0, data0, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    VXC_DP2x8(tmpDst1, data1, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    _viv_asm(COPY, dst0, tmpDst0, 16);\n\
    _viv_asm(COPY, dst1, tmpDst1, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define DEPTH2SPACE_CRD_F16_TO_QINT_BLK2(src1_type_name, write_type) \\\n\
__kernel void depth2space_crd_F16to##src1_type_name##_blk2( \\\n\
    image2d_array_t input, image2d_array_t output, int block_size) \\\n\
{ \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz); \\\n\
    int4 coord_in = coord_out >> 1; \\\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4; \\\n\
    coord_in.w = coord_in.z + 1; \\\n\
    vxc_short8 src0, src1, data0, data1; \\\n\
    vxc_half8 tmpDst0, tmpDst1; \\\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz, \\\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww, \\\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8); \\\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8); \\\n\
 \\\n\
    write_type  dst; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    _viv_asm(COPY, tmpDst0, data0, 16); \\\n\
    _viv_asm(COPY, tmpDst1, data1, 16); \\\n\
    VXC_DP2x8(dst, tmpDst0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_DP2x8(dst, tmpDst1, ms0, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
DEPTH2SPACE_CRD_F16_TO_QINT_BLK2(U8, vxc_uchar16)\n\
DEPTH2SPACE_CRD_F16_TO_QINT_BLK2(I8, vxc_char16)\n\
\n\
__kernel void depth2space_crd_F16toI16_blk2(\n\
    image2d_array_t input, image2d_array_t output, int block_size)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord_out = (int4)(get_global_id(0), gidy, gidz, gidz);\n\
    int4 coord_in = coord_out >> 1;\n\
    coord_in.z = (gidy & 1) * 2 + gidz * 4;\n\
    coord_in.w = coord_in.z + 1;\n\
    vxc_short8 src0, src1, data0, data1, dst0, dst1;\n\
    vxc_half8 tmpDst0, tmpDst1;\n\
    VXC_ReadImage2DArray(src0, input, coord_in.xyzz,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input, coord_in.xyww,\n\
             VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(data0, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_lo_2x8);\n\
    VXC_DP2x8(data1, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniDepth2SpaceF16Blk2_hi_2x8);\n\
    _viv_asm(COPY, tmpDst0, data0, 16);\n\
    _viv_asm(COPY, tmpDst1, data1, 16);\n\
    VXC_DP2x8(dst0, tmpDst0, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
    VXC_DP2x8(dst1, tmpDst1, ms0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 1), uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.x += 8;\n\
    VXC_WriteImage2DArray(output, coord_out, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of depth2space_crd_vx*/

static const char depthwise_conv1d_src0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_align8_step0_16x1;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_align8_step1_16x1;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
_viv_uniform int kernel_size_x16;\n\
_viv_uniform int kernel_size_x8;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_KN_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int2 coord_in = (int2)(get_global_id(0) * stride - pad, get_global_id(1));\n\
int4 coord = (int4)(get_global_id(0), 0, 0, get_global_id(1));\n\
vxc_uchar4 zp;\n\
vxc_uchar16 src;\n\
vxc_uchar16 w;\n\
vxc_short8 coef;\n\
int4 sum, sum0;\n\
\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
sum = read_imagei(bias, coord.wz);\n\
\n\
while(coord.y < kernel_size_x16)\n\
{\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(w, weight, coord.yw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step0_16x1);\n\
    sum.x += sum0.x;\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step1_16x1);\n\
    sum.x += sum0.x;\n\
    coord_in.x += 16;\n\
    coord.y += 16;\n\
}\n\
\n\
if (kernel_size_x8)\n\
{\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(w, weight, coord.yw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step0_16x1);\n\
    sum.x += sum0.x;\n\
}\n\
float4 result = convert_float4(sum.x) * scale + outputZP;\n\
uchar4 dst = convert_uchar4_sat(result);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_KN_D2(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int2 coord_in = (int2)(get_global_id(0) * stride - pad, get_global_id(1));\n\
int4 coord = (int4)(get_global_id(0), 0, 0, get_global_id(1));\n\
vxc_uchar4 zp;\n\
vxc_uchar16 src;\n\
vxc_uchar16 w;\n\
vxc_short8 coef;\n\
int4 sum, sum0;\n\
\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
sum = read_imagei(bias, coord.wz);\n\
\n\
while(coord.y < kernel_size_x8)\n\
{\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(w, weight, coord.yw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(coef, w, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
    VXC_DP16x1(sum0, src, coef, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\\\n\
               uniU8ConvS16_align8_step0_16x1);\n\
    sum.x += sum0.x;\n\
    coord_in.x += 16;\n\
    coord.y += 8;\n\
}\n\
\n\
float4 result = convert_float4(sum.x) * scale + outputZP;\n\
uchar4 dst = convert_uchar4_sat(result);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of depthwise_conv1d_src0_vx*/

static const char depthwise_conv1d_src1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe0_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe1_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe2_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe3_8x2b;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe4_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe5_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe6_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe7_8x2b;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K40_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
vxc_uchar32 src0, src1;\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w0, w1, w2;\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src0.hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src0.lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src1.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(s2, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
int4 coord = (int4)(get_global_id(0), 16, 32, get_global_id(1));\n\
VXC_ReadImage(w0, weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w1, weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w2, weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w0, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w0, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
src1.hi = src0.lo;\n\
VXC_DP2x8(coef, w1, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w1, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
src0.hi = src1.lo;\n\
src0.lo = s2;\n\
VXC_DP2x8(coef, w2, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K56_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
             int              pad,\n\
             int              stride,\n\
             int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
vxc_uchar32 src0, src1, src2;\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w[4];\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src0.hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src0.lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src1.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src1.hi = src0.lo;\n\
VXC_ReadImage(src2.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src2.hi = src1.lo;\n\
int4 coord = (int4)(get_global_id(0), 16, 48, get_global_id(1));\n\
VXC_ReadImage(w[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[2], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[3], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src2.lo, src2.hi, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of depthwise_conv1d_src1_vx*/

static const char depthwise_conv1d_src2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe0_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe1_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe2_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe3_8x2b;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe4_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe5_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe6_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe7_8x2b;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K64_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
int              pad,\n\
int              stride,\n\
int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
vxc_uchar32 src0, src1, src2, src3;\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w[4];\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src0.hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src0.lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src1.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src1.hi = src0.lo;\n\
VXC_ReadImage(src2.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src2.hi = src1.lo;\n\
coord_in.y += 16;\n\
VXC_ReadImage(src3.lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0), \\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src3.hi = src2.lo;\n\
int4 coord = (int4)(get_global_id(0), 16, 48, get_global_id(1));\n\
VXC_ReadImage(w[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[2], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[3], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src1.hi, src1.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src2.hi, src2.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src3.hi, src3.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K80_D1(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
int              pad,\n\
int              stride,\n\
int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 80, get_global_id(1), 0);\n\
vxc_uchar32 src[5];\n\
vxc_uchar16 s0, s1, s2;\n\
vxc_uchar16 w[5];\n\
int4 sum, sumB;\n\
sum = read_imagei(bias, coord_in.zw);\n\
VXC_ReadImage(src[0].hi, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(src[0].lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord_in.x += 32;\n\
VXC_ReadImage(src[1].lo, input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[1].hi = src[0].lo;\n\
VXC_ReadImage(src[2].lo, input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[2].hi = src[1].lo;\n\
VXC_ReadImage(src[3].lo, input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[3].hi = src[2].lo;\n\
VXC_ReadImage(src[4].lo, input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
src[4].hi = src[3].lo;\n\
int4 coord = (int4)(get_global_id(0), 16, 64, get_global_id(1));\n\
VXC_ReadImage(w[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord.y += 16;\n\
VXC_ReadImage(w[2], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[3], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(w[4], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
sum = sum.xxxx;\n\
sumB = sum.xxxx;\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
vxc_short8 coef;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[0].hi, src[0].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum += sum0;\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum += sum1;\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[1].hi, src[1].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[2].hi, src[2].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[3].hi, src[3].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
VXC_DP2x8(coef, w[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sum  += sum0;\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
VXC_DP2x8(coef, w[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe2_8x2b);\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe3_8x2b);\n\
sum  += sum1;\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe6_8x2b);\n\
VXC_DP8x2_b(sum1, src[4].hi, src[4].lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe7_8x2b);\n\
sumB += sum1;\n\
\n\
float4 result0 = convert_float4(sum) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of depthwise_conv1d_src2_vx*/

static const char depthwise_conv1d_src3_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe0_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe1_8x2b;\n\
_viv_uniform VXC_512Bits uniU8SubZp_lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZp_hi_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe4_8x2b;\n\
_viv_uniform VXC_512Bits uniU8ConvS16_Stpe5_8x2b;\n\
_viv_uniform int weightZP;\n\
_viv_uniform float outputZP;\n\
_viv_uniform float scale;\n\
\n\
__kernel void vxDW_Conv1D_U8toU8_K88_D2(\n\
__read_only  image2d_array_t  input,\n\
__read_only  image2d_array_t  weight,\n\
__read_only  image2d_t        bias,\n\
__write_only image2d_array_t  output,\n\
int              pad,\n\
int              stride,\n\
int              dilation)\n\
{\n\
int4 coord_in = (int4)(get_global_id(0) * stride - pad + 16,\\\n\
get_global_id(0) * stride - pad + 48, get_global_id(1), 0);\n\
\n\
vxc_uchar32 src0, src1;\n\
vxc_uchar16 inData[12];\n\
vxc_uchar16 wData[6];\n\
int4 sumA, sumB;\n\
\n\
sumA = read_imagei(bias, coord_in.zw);\n\
\n\
VXC_ReadImage(inData[0], input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[1], input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[2], input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[3], input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord_in.xy += 64;\n\
VXC_ReadImage(inData[4], input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[5], input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[6], input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[7], input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord_in.xy += 64;\n\
VXC_ReadImage(inData[8], input, coord_in.xz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[9], input, coord_in.xz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[10], input, coord_in.yz, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(inData[11], input, coord_in.yz, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
int4 coord = (int4)(get_global_id(0), 16, 48, get_global_id(1));\n\
\n\
VXC_ReadImage(wData[0], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[1], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[2], weight, coord.zw, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[3], weight, coord.zw, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
coord.yz += 64;\n\
VXC_ReadImage(wData[4], weight, coord.yw, VXC_5BITOFFSET_XY(-16, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
VXC_ReadImage(wData[5], weight, coord.yw, VXC_5BITOFFSET_XY(0, 0),\\\n\
VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
sumA = sumA.xxxx;\n\
sumB = sumA;\n\
\n\
int4 sum0, sum1;\n\
vxc_uchar4 zp;\n\
_viv_asm(COPY, zp, weightZP, 4);\n\
\n\
vxc_short8 coef;\n\
src0.hi = inData[0];\n\
src0.lo = inData[1];\n\
\n\
VXC_DP2x8(coef, wData[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = src0.lo;\n\
src0.lo = inData[2];\n\
\n\
VXC_DP2x8(coef, wData[0], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[2];\n\
src0.lo = inData[3];\n\
VXC_DP2x8(coef, wData[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = src0.lo;\n\
src0.lo = inData[4];\n\
\n\
VXC_DP2x8(coef, wData[1], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[4];\n\
src0.lo = inData[5];\n\
\n\
VXC_DP2x8(coef, wData[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = src0.lo;\n\
src0.lo = inData[6];\n\
\n\
VXC_DP2x8(coef, wData[2], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[6];\n\
src0.lo = inData[7];\n\
VXC_DP2x8(coef, wData[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = src0.lo;\n\
src0.lo = inData[8];\n\
VXC_DP2x8(coef, wData[3], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
\n\
src0.hi = inData[8];\n\
src0.lo = inData[9];\n\
VXC_DP2x8(coef, wData[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = src0.lo;\n\
src0.lo = inData[10];\n\
VXC_DP2x8(coef, wData[4], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_hi_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
src0.hi = inData[10];\n\
src0.lo = inData[11];\n\
VXC_DP2x8(coef, wData[5], zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniU8SubZp_lo_2x8);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe0_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe1_8x2b);\n\
sumA += sum0;\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe4_8x2b);\n\
VXC_DP8x2_b(sum0, src0.hi, src0.lo, coef, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0), uniU8ConvS16_Stpe5_8x2b);\n\
sumB += sum0;\n\
float4 result0 = convert_float4(sumA) * scale + outputZP;\n\
float4 result1 = convert_float4(sumB) * scale + outputZP;\n\
int4 dst0 = convert_int4(result0);\n\
int4 dst1 = convert_int4(result1);\n\
vxc_uchar16 dst;\n\
VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtractInteger_2x8);\n\
VXC_WriteImage(output, coord.xw, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of depthwise_conv1d_src3_vx*/

static const char detect_post_box_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform VXC_512Bits uniDataMerge_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZptoF32Conv0_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZptoF32Conv1_4x4;\n\
_viv_uniform float logE;\n\
_viv_uniform int   input0_ZP;\n\
_viv_uniform int   input1_ZP;\n\
\n\
float exp_(float x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
__kernel void detect_post_box_F32_F32toF32(\n\
     __read_only image2d_array_t   input0,\n\
           __read_only image2d_t   input1,\n\
    __write_only image2d_array_t   output,\n\
                           float   inv_scale_y,\n\
                           float   inv_scale_x,\n\
                           float   inv_scale_h,\n\
                           float   inv_scale_w)\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    float4 dst;\n\
    float4 tmp0, tmp1, tmp2, tmp3;\n\
    uint4  tmp5, tmp6, tmp7;\n\
    src0 = read_imagef(input0, coord);\n\
    src1 = read_imagef(input1, coord.xy);\n\
    tmp0.x  = src1.x + src1.z * src0.x * inv_scale_y;\n\
    tmp0.y  = src1.y + src1.w * src0.y * inv_scale_x;\n\
    tmp1.x = src1.z * exp_(src0.z * inv_scale_h) * 0.5f;\n\
    tmp1.y = src1.w * exp_(src0.w * inv_scale_w) * 0.5f;\n\
    tmp2   = tmp0 - tmp1;\n\
    tmp3   = tmp0 + tmp1;\n\
    _viv_asm(COPY, tmp5, tmp2, 16);\n\
    _viv_asm(COPY, tmp6, tmp3, 16);\n\
    VXC_DP4x4(tmp7, tmp5, tmp6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniDataMerge_4x4);\n\
    _viv_asm(COPY, dst, tmp7, 16);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void detect_post_box_U8_U8toF32(\n\
     __read_only image2d_array_t   input0,\n\
     __read_only image2d_array_t   input1,\n\
    __write_only image2d_array_t   output,\n\
                           float   inv_scale_y,\n\
                           float   inv_scale_x,\n\
                           float   inv_scale_h,\n\
                           float   inv_scale_w)\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    float4 dst;\n\
    float4 tmp0, tmp1, tmp2, tmp3;\n\
    vxc_uchar8 in0 = 0, in1 = 0;\n\
    vxc_short8 zp0 = (short)input0_ZP;\n\
    vxc_short8 zp1 = (short)input1_ZP;\n\
    VXC_ReadImage2DArray(in0, input0, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(src0, in0, zp0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniU8SubZptoF32Conv0_4x4);\n\
    VXC_DP4x4(src1, in1, zp1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniU8SubZptoF32Conv1_4x4);\n\
    tmp0.x  = src1.x + src1.z * src0.x * inv_scale_y;\n\
    tmp0.y  = src1.y + src1.w * src0.y * inv_scale_x;\n\
    tmp1.x = src1.z * exp_(src0.z * inv_scale_h) * 0.5f;\n\
    tmp1.y = src1.w * exp_(src0.w * inv_scale_w) * 0.5f;\n\
    dst.xy   = tmp0.xy - tmp1.xy;\n\
    dst.zw   = tmp0.xy + tmp1.xy;\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of detect_post_box_vx*/

static const char eltwise_unary_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float alpha;\n\
_viv_uniform float beta;\n\
\n\
float4 eltwise_unary_sin(float4 x)\n\
{\n\
    return native_sin(x);\n\
}\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
float4 eltwise_unary_exp(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
#define rlogE    (0.693147182f)\n\
float4 eltwise_unary_log(float4 x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 eltwise_unary_elu(float4 val)\n\
{\n\
    float4 x = val * logE;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val;\n\
}\n\
\n\
float4 eltwise_unary_neg(float4 x)\n\
{\n\
    return x * -1;\n\
}\n\
\n\
float4 eltwise_unary_hard_sigmoid(float4 x)\n\
{\n\
    x = alpha * x + beta;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
float4 _softrelu(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    x += 1;\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 _tanh(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return (2 * x - 1);\n\
}\n\
\n\
float4 eltwise_unary_mish(float4 x)\n\
{\n\
    float4 y = _softrelu(x);\n\
    x = x * _tanh(y);\n\
    return x;\n\
}\n\
\n\
float4 eltwise_unary_round(float4 x)\n\
{\n\
    return convert_float4(convert_int4_rte(x));\n\
}\n\
\n\
#define MUL2_RSQRTPI    (1.1283791670955126f)\n\
float erf_eval(float x)\n\
{\n\
    float res = 0;\n\
    float tmp = x;\n\
    float factorial = 1;\n\
    float x_pow = x;\n\
    float one = 1.0f;\n\
    float n = 1;\n\
\n\
    if (x <= -3)\n\
        return -1;\n\
    else if(x >= 3)\n\
        return 1;\n\
\n\
    while (fabs(tmp) > 1e-5)\n\
    {\n\
        res += tmp;\n\
\n\
        factorial *= n;\n\
        one *= -1;\n\
        x_pow *= x * x;\n\
        tmp = one / factorial * x_pow / ( 2 * n + 1);\n\
\n\
        n += 1.0f;\n\
    }\n\
    return res * MUL2_RSQRTPI;\n\
}\n\
#define RSQRT2      (0.70710678118654752440084436210485f)\n\
float4 eltwise_unary_gelu(float4 x)\n\
{\n\
    float4 erf, data;\n\
    data = x * RSQRT2;\n\
    erf.x = erf_eval(data.x);\n\
    erf.y = erf_eval(data.y);\n\
    erf.z = erf_eval(data.z);\n\
    erf.w = erf_eval(data.w);\n\
    x = 0.5f * x * (1 + erf);\n\
\n\
    return x;\n\
}\n\
\n\
#define SQRT_2_RCP_PI  0.7978845834732056f\n\
float4 eltwise_unary_hard_gelu(float4 x)\n\
{\n\
    float4 cdf = 0.5f + 0.5f * _tanh(SQRT_2_RCP_PI *\n\
                        (x + 0.044715f * x * x * x));\n\
    return x * cdf;\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define ELTSISE_UNARY_2D(func_name, src_type_name, dst_type_name, src_type, \\\n\
        src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
    __kernel void func_name##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecB = vecB * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
    vecB = vecB * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//EXP\n\
ELTSISE_UNARY_2D(exp, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(exp, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(exp, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(exp, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(exp, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(exp, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(exp, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(exp, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(exp, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(exp, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//SIN\n\
ELTSISE_UNARY_2D(sin, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(sin, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(sin, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(sin, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(sin, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(sin, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(sin, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(sin, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(sin, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(sin, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//LOG\n\
ELTSISE_UNARY_2D(log, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(log, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(log, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(log, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(log, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(log, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(log, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(log, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(log, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(log, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//ELU\n\
ELTSISE_UNARY_2D(elu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(elu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(elu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(elu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(elu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(elu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(elu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(elu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(elu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(elu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//NEG\n\
ELTSISE_UNARY_2D(neg, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(neg, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(neg, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(neg, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(neg, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(neg, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(neg, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(neg, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(neg, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(neg, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//MISH\n\
ELTSISE_UNARY_2D(mish, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(mish, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(mish, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(mish, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(mish, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(mish, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(mish, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(mish, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(mish, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(mish, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_sigmoid, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//ROUND\n\
ELTSISE_UNARY_2D(round, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(round, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(round, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(round, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(round, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(round, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(round, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(round, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(round, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(round, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//GELU\n\
ELTSISE_UNARY_2D(gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_GELU\n\
ELTSISE_UNARY_2D(hard_gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(hard_gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(hard_gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(hard_gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define ELTSISE_UNARY_BF16_2D(func_name) \\\n\
    __kernel void func_name##_BF16toBF16_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//EXP\n\
ELTSISE_UNARY_BF16_2D(exp)\n\
//SIN\n\
ELTSISE_UNARY_BF16_2D(sin)\n\
//LOG\n\
ELTSISE_UNARY_BF16_2D(log)\n\
//ELU\n\
ELTSISE_UNARY_BF16_2D(elu)\n\
//NEG\n\
ELTSISE_UNARY_BF16_2D(neg)\n\
//MISH\n\
ELTSISE_UNARY_BF16_2D(mish)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_BF16_2D(hard_sigmoid)\n\
//ROUND\n\
ELTSISE_UNARY_BF16_2D(round)\n\
//GELU\n\
ELTSISE_UNARY_BF16_2D(gelu)\n\
//HARD_GELU\n\
ELTSISE_UNARY_BF16_2D(hard_gelu)\n\
"; /* end of eltwise_unary_2d_vx*/

static const char eltwise_unary_3d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float alpha;\n\
_viv_uniform float beta;\n\
\n\
float4 eltwise_unary_sin(float4 x)\n\
{\n\
    return native_sin(x);\n\
}\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
float4 eltwise_unary_exp(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
#define rlogE    (0.693147182f)\n\
float4 eltwise_unary_log(float4 x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 eltwise_unary_elu(float4 val)\n\
{\n\
    float4 x = val * logE;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val;\n\
}\n\
\n\
float4 eltwise_unary_neg(float4 x)\n\
{\n\
    return x * -1;\n\
}\n\
\n\
float4 eltwise_unary_hard_sigmoid(float4 x)\n\
{\n\
    x = alpha * x + beta;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
float4 _softrelu(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    x += 1;\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float4 _tanh(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return (2 * x - 1);\n\
}\n\
\n\
float4 eltwise_unary_mish(float4 x)\n\
{\n\
    float4 y = _softrelu(x);\n\
    x = x * _tanh(y);\n\
    return x;\n\
}\n\
\n\
float4 eltwise_unary_round(float4 x)\n\
{\n\
    return convert_float4(convert_int4_rte(x));\n\
}\n\
\n\
#define MUL2_RSQRTPI    (1.1283791670955126f)\n\
float erf_eval(float x)\n\
{\n\
    float res = 0;\n\
    float tmp = x;\n\
    float factorial = 1;\n\
    float x_pow = x;\n\
    float one = 1.0f;\n\
    float n = 1;\n\
\n\
    if (x <= -3)\n\
        return -1;\n\
    else if(x >= 3)\n\
        return 1;\n\
\n\
    while (fabs(tmp) > 1e-5)\n\
    {\n\
        res += tmp;\n\
\n\
        factorial *= n;\n\
        one *= -1;\n\
        x_pow *= x * x;\n\
        tmp = one / factorial * x_pow / ( 2 * n + 1);\n\
\n\
        n += 1.0f;\n\
    }\n\
    return res * MUL2_RSQRTPI;\n\
}\n\
#define RSQRT2      (0.70710678118654752440084436210485f)\n\
float4 eltwise_unary_gelu(float4 x)\n\
{\n\
    float4 erf, data;\n\
    data = x * RSQRT2;\n\
    erf.x = erf_eval(data.x);\n\
    erf.y = erf_eval(data.y);\n\
    erf.z = erf_eval(data.z);\n\
    erf.w = erf_eval(data.w);\n\
    x = 0.5f * x * (1 + erf);\n\
\n\
    return x;\n\
}\n\
\n\
#define SQRT_2_RCP_PI  0.7978845834732056f\n\
float4 eltwise_unary_hard_gelu(float4 x)\n\
{\n\
    float4 cdf = 0.5f + 0.5f * _tanh(SQRT_2_RCP_PI *\n\
                        (x + 0.044715f * x * x * x));\n\
    return x * cdf;\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define ELTSISE_UNARY_3D(func_name, src_type_name, dst_type_name, src_type, \\\n\
                src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
__kernel void func_name##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecB = vecB * inputScale + inputTail; \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
    vecB = vecB * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//EXP\n\
ELTSISE_UNARY_3D(exp, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(exp, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(exp, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(exp, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(exp, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(exp, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(exp, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(exp, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(exp, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(exp, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//SIN\n\
ELTSISE_UNARY_3D(sin, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(sin, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(sin, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(sin, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(sin, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(sin, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(sin, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(sin, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(sin, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(sin, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//LOG\n\
ELTSISE_UNARY_3D(log, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(log, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(log, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(log, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(log, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(log, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(log, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(log, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(log, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(log, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//ELU\n\
ELTSISE_UNARY_3D(elu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(elu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(elu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(elu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(elu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(elu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(elu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(elu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(elu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(elu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//NEG\n\
ELTSISE_UNARY_3D(neg, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(neg, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(neg, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(neg, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(neg, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(neg, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(neg, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(neg, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(neg, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(neg, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//MISH\n\
ELTSISE_UNARY_3D(mish, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(mish, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(mish, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(mish, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(mish, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(mish, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(mish, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(mish, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(mish, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(mish, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_sigmoid, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//ROUND\n\
ELTSISE_UNARY_3D(round, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(round, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(round, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(round, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(round, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(round, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(round, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(round, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(round, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(round, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//GELU\n\
ELTSISE_UNARY_3D(gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
//HARD_GELU\n\
ELTSISE_UNARY_3D(hard_gelu, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_gelu, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_gelu, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(hard_gelu, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(hard_gelu, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(hard_gelu, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
#define ELTSISE_UNARY_BF16(func_name) \\\n\
    __kernel void func_name##_BF16toBF16( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              type, \\\n\
                 float            _alpha, \\\n\
                 float            _beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    float4 vecB; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecA = eltwise_unary_##func_name(vecA); \\\n\
    vecB = eltwise_unary_##func_name(vecB); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//EXP\n\
ELTSISE_UNARY_BF16(exp)\n\
//SIN\n\
ELTSISE_UNARY_BF16(sin)\n\
//LOG\n\
ELTSISE_UNARY_BF16(log)\n\
//ELU\n\
ELTSISE_UNARY_BF16(elu)\n\
//NEG\n\
ELTSISE_UNARY_BF16(neg)\n\
//MISH\n\
ELTSISE_UNARY_BF16(mish)\n\
//HARD_SIGMOID\n\
ELTSISE_UNARY_BF16(hard_sigmoid)\n\
//ROUND\n\
ELTSISE_UNARY_BF16(round)\n\
//GELU\n\
ELTSISE_UNARY_BF16(gelu)\n\
//HARD_GELU\n\
ELTSISE_UNARY_BF16(hard_gelu)"; /* end of eltwise_unary_3d_vx*/

static const char erf_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define MUL2_RSQRTPI    (1.1283791670955126f)\n\
float eltwise_unary_erf(float _x)\n\
{\n\
    float x = clamp(_x, -2, 2);\n\
    float res = 0;\n\
    float tmp = x;\n\
    float factorial = 1;\n\
    float x_pow = x;\n\
    float one = 1.0f;\n\
    float n = 1;\n\
\n\
    while (fabs(tmp) > 1e-5)\n\
    {\n\
        res += tmp;\n\
\n\
        factorial *= n;\n\
        one *= -1;\n\
        x_pow *= x * x;\n\
        tmp = one / factorial * x_pow / ( 2 * n + 1);\n\
\n\
        n += 1.0f;\n\
    }\n\
    return res * MUL2_RSQRTPI;\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
\n\
#define ELTSISE_UNARY_2D(func_name, src_type_name, dst_type_name, src_type, \\\n\
        src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
    __kernel void func_name##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecA.x = eltwise_unary_##func_name(vecA.x); \\\n\
    vecA.y = eltwise_unary_##func_name(vecA.y); \\\n\
    vecA.z = eltwise_unary_##func_name(vecA.z); \\\n\
    vecA.w = eltwise_unary_##func_name(vecA.w); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//ERF\n\
ELTSISE_UNARY_2D(erf, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(erf, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(erf, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(erf, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(erf, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_2D(erf, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(erf, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_2D(erf, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_2D(erf, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_2D(erf, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define ELTSISE_UNARY_BF16_2D(func_name) \\\n\
    __kernel void func_name##_BF16toBF16_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    vecA.x = eltwise_unary_##func_name(vecA.x); \\\n\
    vecA.y = eltwise_unary_##func_name(vecA.y); \\\n\
    vecA.z = eltwise_unary_##func_name(vecA.z); \\\n\
    vecA.w = eltwise_unary_##func_name(vecA.w); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//EXP\n\
ELTSISE_UNARY_BF16_2D(erf)\n\
\n\
#define ELTSISE_UNARY_3D(func_name, src_type_name, dst_type_name, src_type, \\\n\
    src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
__kernel void func_name##_##src_type_name##to##dst_type_name( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output \\\n\
) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
 \\\n\
    float4 vecA; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    vecA = vecA * inputScale + inputTail; \\\n\
    vecA.x = eltwise_unary_##func_name(vecA.x); \\\n\
    vecA.y = eltwise_unary_##func_name(vecA.y); \\\n\
    vecA.z = eltwise_unary_##func_name(vecA.z); \\\n\
    vecA.w = eltwise_unary_##func_name(vecA.w); \\\n\
    vecA = vecA * outputScale + outputZP; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//ERF\n\
ELTSISE_UNARY_3D(erf, F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(erf, F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(erf, F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(erf, F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(erf, I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
ELTSISE_UNARY_3D(erf, I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(erf, U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
ELTSISE_UNARY_3D(erf, U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
ELTSISE_UNARY_3D(erf, I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
ELTSISE_UNARY_3D(erf, I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
\n\
#define ELTSISE_UNARY_BF16_3D(func_name) \\\n\
    __kernel void func_name##_BF16toBF16( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    VXC_ReadImage2DArray(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 vecA; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    vecA.x = eltwise_unary_##func_name(vecA.x); \\\n\
    vecA.y = eltwise_unary_##func_name(vecA.y); \\\n\
    vecA.z = eltwise_unary_##func_name(vecA.z); \\\n\
    vecA.w = eltwise_unary_##func_name(vecA.w); \\\n\
 \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//ERF\n\
ELTSISE_UNARY_BF16_3D(erf)"; /* end of erf_vx*/

static const char extra_ending_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void extra_ending_I16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void extra_ending_F16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void extra_ending_I8\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void extra_ending_U8\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 data;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of extra_ending_vx*/

static const char floordiv_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertFstToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecToFp32_4x4;\n\
\n\
_viv_uniform float in_scale0;\n\
_viv_uniform float in_scale1;\n\
_viv_uniform float out_scale;\n\
_viv_uniform float in0Tail;\n\
_viv_uniform float in1Tail;\n\
_viv_uniform float out_zp;\n\
\n\
#define FLOORDIV_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                      IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    save_type data; \\\n\
    read_type read_data0, read_data1; \\\n\
    copy_type tmpData0, tmpData1; \\\n\
    vxc_float4 in0Val1, in0Val2, in1Val1, in1Val2; \\\n\
    vxc_float4 tmpVal1, tmpVal2; \\\n\
    dst_type tmpOut1, tmpOut2; \\\n\
    read_fun(read_data0, input0, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmpData0, read_data0, 16); \\\n\
    read_fun(read_data1, input1, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, tmpData1, read_data1, 16); \\\n\
    VXC_DP4x4(in0Val1, tmpData0, tmpData0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
    VXC_DP4x4(in0Val2, tmpData0, tmpData0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
    VXC_DP4x4(in1Val1, tmpData1, tmpData1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstToFp32_4x4); \\\n\
    VXC_DP4x4(in1Val2, tmpData1, tmpData1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecToFp32_4x4); \\\n\
    in0Val1 = in0Val1 * IN0_SCALE + IN0_TAIL; \\\n\
    in0Val2 = in0Val2 * IN0_SCALE + IN0_TAIL; \\\n\
    in1Val1 = in1Val1 * IN1_SCALE + IN1_TAIL; \\\n\
    in1Val2 = in1Val2 * IN1_SCALE + IN1_TAIL; \\\n\
    tmpVal1 = floor(in0Val1 / in1Val1) * OUT_SCALE + OUT_OFFSET; \\\n\
    tmpVal2 = floor(in0Val2 / in1Val2) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, tmpOut1, tmpVal1); \\\n\
    _viv_asm(conv_mode, tmpOut2, tmpVal2); \\\n\
    VXC_DP2x8(data, tmpOut1, tmpOut2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
\n\
#define TENSOR_FLOORDIV(src0_name, src1_name, dst_name, dst_type, save_type, read_type, copy_type, \\\n\
                    conv_mode, IN0_SCALE, IN0_TAIL, IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET) \\\n\
__kernel void floordiv_##src0_name##src1_name##to##dst_name \\\n\
    ( \\\n\
    image2d_array_t input0, \\\n\
    image2d_array_t input1, \\\n\
    image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    FLOORDIV_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                  IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
\n\
TENSOR_FLOORDIV(F16, F16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_half8, CONV, 1, 0, 1, 0, 1, 0)\n\
TENSOR_FLOORDIV(F16, F16, I16, short4, vxc_short8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV(F16, F16, I8,  char4, vxc_char8, vxc_short8,\\\n\
                vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV(F16, F16, U8,  uchar4, vxc_uchar8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
\n\
TENSOR_FLOORDIV(I16, I16, I16, short4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV_SAT_RTE, in_scale0, 0, in_scale1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV(I16, I16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV, in_scale0, 0, in_scale1, 0, 1, 0)\n\
\n\
TENSOR_FLOORDIV(I8, I8, I8, char4, vxc_char8, vxc_char16,\\\n\
                vxc_char16, CONV_SAT_RTE, in_scale0, 0, in_scale1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV(I8, I8, F16, half4, vxc_short8, vxc_char16,\\\n\
                vxc_char16, CONV, in_scale0, 0, in_scale1, 0, 1, 0)\n\
\n\
TENSOR_FLOORDIV(U8, U8, U8,  uchar4, vxc_uchar8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV(U8, U8, F16, half4, vxc_short8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
\n\
\n\
#define TENSOR_FLOORDIV_2D(src0_name, src1_name, dst_name, dst_type, save_type, read_type, copy_type, \\\n\
    conv_mode, IN0_SCALE, IN0_TAIL, IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET) \\\n\
__kernel void floordiv_##src0_name##src1_name##to##dst_name##_2D \\\n\
    ( \\\n\
    image2d_array_t input0, \\\n\
    image2d_array_t input1, \\\n\
    image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    FLOORDIV_PROCESS(dst_type, save_type, read_type, copy_type, conv_mode, IN0_SCALE, IN0_TAIL,\\\n\
                  IN1_SCALE, IN1_TAIL, OUT_SCALE, OUT_OFFSET, VXC_ReadImage, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
TENSOR_FLOORDIV_2D(F16, F16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_half8, CONV, 1, 0, 1, 0, 1, 0)\n\
TENSOR_FLOORDIV_2D(F16, F16, I16, short4, vxc_short8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV_2D(F16, F16, I8,  char4, vxc_char8, vxc_short8,\\\n\
                vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV_2D(F16, F16, U8,  uchar4, vxc_uchar8, vxc_short8,\\\n\
               vxc_half8, CONV_SAT_RTE, 1, 0, 1, 0, out_scale, out_zp)\n\
\n\
TENSOR_FLOORDIV_2D(I16, I16, I16, short4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV_SAT_RTE, in_scale0, 0, in_scale1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV_2D(I16, I16, F16, half4, vxc_short8, vxc_short8,\\\n\
                vxc_short8, CONV, in_scale0, 0, in_scale1, 0, 1, 0)\n\
\n\
TENSOR_FLOORDIV_2D(I8, I8, I8, char4, vxc_char8, vxc_char16,\\\n\
                vxc_char16, CONV_SAT_RTE, in_scale0, 0, in_scale1, 0, out_scale, 0)\n\
TENSOR_FLOORDIV_2D(I8, I8, F16, half4, vxc_short8, vxc_char16,\\\n\
                vxc_char16, CONV, in_scale0, 0, in_scale1, 0, 1, 0)\n\
\n\
TENSOR_FLOORDIV_2D(U8, U8, U8,  uchar4, vxc_uchar8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV_SAT_RTE, in_scale0, in0Tail, in_scale1, in1Tail, out_scale, out_zp)\n\
TENSOR_FLOORDIV_2D(U8, U8, F16, half4, vxc_short8, vxc_uchar16,\\\n\
                vxc_uchar16, CONV, in_scale0, in0Tail, in_scale1, in1Tail, 1, 0)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define FLOORDIV_BF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 read_data0, read_data1, vec0; \\\n\
    vxc_float4 in0Val1, in0Val2, in1Val1, in1Val2; \\\n\
    vxc_float4 tmpVal1, tmpVal2; \\\n\
    vxc_ushort8 dst0, dst1; \\\n\
    vxc_ushort8 vect; \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    read_fun(read_data0, input0, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(vec0, read_data0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, in0Val1, vec0, 16); \\\n\
    VXC_DP2x8(vec0, read_data0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, in0Val2, vec0, 16); \\\n\
    read_fun(read_data1, input1, coord, VXC_5BITOFFSET_XY(0,0), \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(vec0, read_data1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, in1Val1, vec0, 16); \\\n\
    VXC_DP2x8(vec0, read_data1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, in1Val2, vec0, 16); \\\n\
    tmpVal1 = floor(in0Val1 / in1Val1); \\\n\
    tmpVal2 = floor(in0Val2 / in1Val2); \\\n\
    _viv_asm(COPY, dst0, tmpVal1, 16); \\\n\
    _viv_asm(COPY, dst1, tmpVal2, 16); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void floordiv_BF16BF16toBF16\n\
    (\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    FLOORDIV_BF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray);\n\
}\n\
\n\
__kernel void floordiv_BF16BF16toBF16_2D\n\
    (\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    FLOORDIV_BF16_PROCESS(VXC_ReadImage, VXC_WriteImage);\n\
}"; /* end of floordiv_vx*/

static const char gather_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
_viv_uniform VXC_512Bits uniExtraCopyDpKeepinEvis_2x8;\n\
\n\
__kernel void gather_I8toI8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_U8toU8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_I16toI16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
\n\
\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
\n\
\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_I8toI8_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_char16 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_U8toU8_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_uchar16 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_I16toI16_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_F16toF16_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_short8 src, dst;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniExtraCopyDpKeepinEvis_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_vx*/

static const char gather_array_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
_viv_uniform VXC_512Bits uniExtraCopyDpKeepinEvis_2x8;\n\
\n\
__kernel void gather_I8toI8_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    __global vxc_char16* data_ptr = (__global vxc_char16*)input_ptr;\n\
    vxc_char16 src = data_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global vxc_char16* dst_ptr = (__global vxc_char16*)output_ptr;\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_U8toU8_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 1);\n\
    Image img2 = create_image_from_image2d(output, 1);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    __global vxc_uchar16* data_ptr = (__global vxc_uchar16*)input_ptr;\n\
    vxc_uchar16 src = data_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global vxc_uchar16* dst_ptr = (__global vxc_uchar16*)output_ptr;\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_I16toI16_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
\n\
\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    __global vxc_short8* data_ptr = (__global vxc_short8*)input_ptr;\n\
    vxc_short8 src = data_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global vxc_short8* dst_ptr = (__global vxc_short8*)output_ptr;\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
__kernel void gather_F16toF16_array(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    Image img1 = create_image_from_image2d(input0, 2);\n\
    Image img2 = create_image_from_image2d(output, 2);\n\
    uchar* input_ptr = get_image_ptr_from_coord(img1, coord_in.zw);\n\
    __global vxc_short8* data_ptr = (__global vxc_short8*)input_ptr;\n\
    vxc_short8 src = data_ptr[0];\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord);\n\
    __global vxc_short8* dst_ptr = (__global vxc_short8*)output_ptr;\n\
    dst_ptr[0] = src;\n\
}\n\
\n\
#define GATHER_AXIS0_ARRAY(src0_type_name, read_type, data_type, write_type) \\\n\
__kernel void gather_##src0_type_name##to##src0_type_name##_axis0_array( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num \\\n\
    ) \\\n\
{ \\\n\
    Image img0 = create_image_from_image2d(input0, 1); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    Image img2 = create_image_from_image2d(output, 1); \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    uchar* index_ptr = get_image_ptr_from_coord(img1, coord.xz); \\\n\
    __global int* index = (__global int*)index_ptr; \\\n\
    int4 indices = vload4(0, index); \\\n\
 \\\n\
    read_type src, dst; \\\n\
 \\\n\
    uchar* input_ptr = get_image_ptr_from_coord(img0, coord.zy); \\\n\
    uchar* output_ptr = get_image_ptr_from_coord(img2, coord.xy); \\\n\
    __global data_type* data_ptr = (__global data_type*)input_ptr; \\\n\
    __global write_type* out_ptr = (__global write_type*)output_ptr; \\\n\
    src.s0 = data_ptr[indices.x]; \\\n\
    src.s1 = data_ptr[indices.y]; \\\n\
    src.s2 = data_ptr[indices.z]; \\\n\
    src.s3 = data_ptr[indices.w]; \\\n\
 \\\n\
    VXC_DP2x8(dst, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
                     uniExtraCopyDpKeepinEvis_2x8); \\\n\
    out_ptr[0] = dst.s0123; \\\n\
}\n\
GATHER_AXIS0_ARRAY(U8, vxc_uchar16, uchar, vxc_uchar4)\n\
GATHER_AXIS0_ARRAY(I8, vxc_char16,  char, vxc_char4)\n\
GATHER_AXIS0_ARRAY(I16, vxc_short8, short, vxc_short4)\n\
GATHER_AXIS0_ARRAY(F16, vxc_short8, short, vxc_short4)"; /* end of gather_array_vx*/

static const char gather_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int indices_num;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define GATHER_8BITS_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gather_##src0_type_name##toF16( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0); \\\n\
    int4 indice = read_imagei(input1, coord_in.xy); \\\n\
    coord_in.w = gidz * axis_num + indice.x; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy); \\\n\
    vxc_half8  src0, src1; \\\n\
    vxc_short8 dst0, dst1; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0, VXC_MODIFIER(0,7,0, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    VXC_DP2x8(src1,src,ms0, VXC_MODIFIER(0,7,8, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    _viv_asm(COPY, dst1, src1, 16); \\\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord.x += 8; \\\n\
    VXC_WriteImage(output, coord, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_8BITS_TO_F16(U8, vxc_uchar16)\n\
GATHER_8BITS_TO_F16(I8, vxc_char16)\n\
\n\
#define GATHER_F16_TO_QINT(src1_type_name, write_type) \\\n\
__kernel void gather_F16to##src1_type_name( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0); \\\n\
 \\\n\
    int4 indice = read_imagei(input1, coord_in.xy); \\\n\
    coord_in.w = gidz * axis_num + indice.x; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy); \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_F16_TO_QINT(U8, vxc_uchar16)\n\
GATHER_F16_TO_QINT(I8, vxc_char16)\n\
GATHER_F16_TO_QINT(I16, vxc_short8)\n\
\n\
__kernel void gather_I16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in.zw, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
\n\
    vxc_half8  src0;\n\
    vxc_short8 dst0;\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
    _viv_asm(COPY, dst0, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define GATHER_8BITS_TO_F16_AXIS0(src0_type_name, read_type) \\\n\
__kernel void gather_##src0_type_name##toF16_axis0( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int4 indices = read_imagei(input1, coord.xx); \\\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1)); \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    indices.x = get_global_id(1); \\\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0, VXC_MODIFIER(0,7,0, VXC_RM_TowardZero,1), uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_8BITS_TO_F16_AXIS0(U8, vxc_uchar16)\n\
GATHER_8BITS_TO_F16_AXIS0(I8, vxc_char16)\n\
\n\
#define GATHER_F16_TO_QINT_AXIS0(src1_type_name, write_type) \\\n\
__kernel void gather_F16to##src1_type_name##_axis0( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int block_num, \\\n\
    int axis_num \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int4 indices = read_imagei(input1, coord.xx); \\\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1)); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    indices.x = get_global_id(1); \\\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_F16_TO_QINT_AXIS0(U8, vxc_uchar16)\n\
GATHER_F16_TO_QINT_AXIS0(I8, vxc_char16)\n\
GATHER_F16_TO_QINT_AXIS0(I16, vxc_short8)\n\
\n\
__kernel void gather_I16toF16_axis0(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 indices = read_imagei(input1, coord.xx);\n\
    int2 coord_in = (int2)(indices.x, get_global_id(1));\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    indices.x = get_global_id(1);\n\
    VXC_ReadImage(src, input0, indices.yx, 0, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.zx, 0, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src, input0, indices.wx, 0, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_half8  src0;\n\
    vxc_short8 dst0;\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
    _viv_asm(COPY, dst0, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_mix_vx*/

static const char gather_nd_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_I8toI8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_I16toI16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    coord.w = indice.x;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_nd_vx*/

static const char gather_nd_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_I8toI8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_U8toU8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_I16toI16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of gather_nd_2d_vx*/

static const char gather_nd_2d_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16toFp16_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt16_2x8;\n\
\n\
#define GATHER_ND_QINT_TO_F16_2D(src0_type_name, read_type) \\\n\
__kernel void gather_nd_##src0_type_name##toF16_2D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord.zy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_QINT_TO_F16_2D(U8, vxc_uchar16)\n\
GATHER_ND_QINT_TO_F16_2D(I8, vxc_char16)\n\
GATHER_ND_QINT_TO_F16_2D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_F16_TO_QINT_2D(src1_type_name, write_type) \\\n\
__kernel void gather_nd_F16to##src1_type_name##_2D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, indice.xy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven,1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord.zy, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_F16_TO_QINT_2D(U8, vxc_uchar16)\n\
GATHER_ND_F16_TO_QINT_2D(I8, vxc_char16)\n\
GATHER_ND_F16_TO_QINT_2D(I16, vxc_short8)\n\
"; /* end of gather_nd_2d_mix_vx*/

static const char gather_nd_3d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void gather_nd_I8toI8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_char16 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_U8toU8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_uchar16 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_I16toI16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    Image img = create_image_from_image2d(input1, 4);\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy);\n\
    int4 indice = ((int4 *)indice_ptr)[0];\n\
\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.w = 0;\n\
\n\
    vxc_short8 src;\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.zy, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of gather_nd_3d_vx*/

static const char gather_nd_3d_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define GATHER_ND_QINT_TO_F16_3D(src0_type_name, read_type) \\\n\
__kernel void gather_nd_##src0_type_name##toF16_3D( \\\n\
    __read_only image2d_array_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
    indice.w = 0; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord.zy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_QINT_TO_F16_3D(U8, vxc_uchar16)\n\
GATHER_ND_QINT_TO_F16_3D(I8, vxc_char16)\n\
GATHER_ND_QINT_TO_F16_3D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_F16_TO_QINT_3D(src1_type_name, write_type) \\\n\
__kernel void gather_nd_F16to##src1_type_name##_3D( \\\n\
    __read_only image2d_array_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    indice.x = indice.x * block_size + gidx; \\\n\
    indice.w = 0; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage2DArray(src, input0, indice, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven,1), uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord.zy, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_F16_TO_QINT_3D(U8, vxc_uchar16)\n\
GATHER_ND_F16_TO_QINT_3D(I8, vxc_char16)\n\
GATHER_ND_F16_TO_QINT_3D(I16, vxc_short8)\n\
\n\
"; /* end of gather_nd_3d_mix_vx*/

static const char gather_nd_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16toFp16_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt16_2x8;\n\
\n\
#define GATHER_ND_QINT_TO_F16_1D(src0_type_name, read_type) \\\n\
__kernel void gather_nd_##src0_type_name##toF16_1D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    coord.w = indice.x; \\\n\
 \\\n\
    read_type src; \\\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_half8  src0; \\\n\
    vxc_short8 dst0; \\\n\
    vxc_ushort8 ms0; \\\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
    VXC_DP2x8(src0,src,ms0,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,1),uniU8MulAndPostShift_0_Lo_2x8); \\\n\
    _viv_asm(COPY, dst0, src0, 16); \\\n\
    VXC_WriteImage(output, coord.zy, dst0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_QINT_TO_F16_1D(U8, vxc_uchar16)\n\
GATHER_ND_QINT_TO_F16_1D(I8, vxc_char16)\n\
GATHER_ND_QINT_TO_F16_1D(I16, vxc_short8)\n\
\n\
#define GATHER_ND_F16_TO_QINT_1D(src1_type_name, write_type) \\\n\
__kernel void gather_nd_F16to##src1_type_name##_1D( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output, \\\n\
    int block_size, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord = (int4)(0, gidy, gidx, 0); \\\n\
    Image img = create_image_from_image2d(input1, 4); \\\n\
    uchar* indice_ptr = get_image_ptr_from_coord(img, coord.xy); \\\n\
    int4 indice = ((int4 *)indice_ptr)[0]; \\\n\
 \\\n\
    coord.w = indice.x; \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input0, coord.zw, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 mp1; \\\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16); \\\n\
    vxc_half8 data; \\\n\
    write_type dst; \\\n\
    _viv_asm(COPY, data, src, 16); \\\n\
    VXC_DP2x8(dst,data,mp1,VXC_MODIFIER(0,7,0,VXC_RM_ToNearestEven, 1),uniConvertFp16toU8_2x8); \\\n\
    VXC_WriteImage(output, coord.zy, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GATHER_ND_F16_TO_QINT_1D(U8, vxc_uchar16)\n\
GATHER_ND_F16_TO_QINT_1D(I8, vxc_char16)\n\
GATHER_ND_F16_TO_QINT_1D(I16, vxc_short8)\n\
\n\
"; /* end of gather_nd_mix_vx*/

static const char get_matrix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float4 theta_1;\n\
_viv_uniform float4 theta_2;\n\
_viv_uniform float4 scale;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
\n\
#define GET_MATRIX_SH_IMPL(name0, in_type, read_func) \\\n\
__kernel void get_matrix_##name0##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 int       has_theta_1_1,  \\\n\
                 int       has_theta_1_2,  \\\n\
                 int       has_theta_1_3,  \\\n\
                 int       has_theta_2_1,  \\\n\
                 int       has_theta_2_2,  \\\n\
                 int       has_theta_2_3,  \\\n\
                 float     theta_1_1,  \\\n\
                 float     theta_1_2,  \\\n\
                 float     theta_1_3,  \\\n\
                 float     theta_2_1,  \\\n\
                 float     theta_2_2,  \\\n\
                 float     theta_2_3,  \\\n\
                 float     i_width, \\\n\
                 float     i_height, \\\n\
                 float     o_width, \\\n\
                 float     o_height \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(0, get_global_id(1)); \\\n\
    float4 matrix0, matrix1; \\\n\
    float4 theta1, theta2; \\\n\
    _viv_asm(COPY, theta1, theta_1, 16); \\\n\
    _viv_asm(COPY, theta2, theta_2, 16); \\\n\
 \\\n\
    if (has_theta_1_1 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta1.x = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_1_2 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta1.y = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_1_3 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta1.z = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_2_1 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta2.x = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_2_2 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta2.y = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    if (has_theta_2_3 == 0) \\\n\
    { \\\n\
        in_type data = read_func(input, coord); \\\n\
        coord.x ++; \\\n\
        theta2.z = convert_float(data.x) * input_scale + input_tail; \\\n\
    } \\\n\
 \\\n\
    matrix0.x = theta2.y * scale.x; \\\n\
    matrix0.z = theta2.x * scale.z; \\\n\
    matrix1.x = ( theta2.z - theta2.y - theta2.x + 1) * i_width * 0.5f; \\\n\
    matrix0.y = theta1.y * scale.w; \\\n\
    matrix0.w = theta1.x * scale.y; \\\n\
    matrix1.y = ( theta1.z - theta1.y - theta1.x + 1) * i_height * 0.5f; \\\n\
    matrix1.zw = 2.0f * matrix0.xy; \\\n\
 \\\n\
    coord.x = 0; \\\n\
    vxc_ushort8 dst; \\\n\
    _viv_asm(COPY, dst, matrix0, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, dst, matrix1, 16); \\\n\
    coord.x = 8; \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GET_MATRIX_SH_IMPL(I16, int4,  read_imagei)\n\
GET_MATRIX_SH_IMPL(I8,  int4,  read_imagei)\n\
GET_MATRIX_SH_IMPL(U8,  uint4, read_imageui)\n\
\n\
__kernel void get_matrix_F16toF32\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 int       has_theta_1_1,\n\
                 int       has_theta_1_2,\n\
                 int       has_theta_1_3,\n\
                 int       has_theta_2_1,\n\
                 int       has_theta_2_2,\n\
                 int       has_theta_2_3,\n\
                 float     theta_1_1,\n\
                 float     theta_1_2,\n\
                 float     theta_1_3,\n\
                 float     theta_2_1,\n\
                 float     theta_2_2,\n\
                 float     theta_2_3,\n\
                 float     i_width,\n\
                 float     i_height,\n\
                 float     o_width,\n\
                 float     o_height\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(1));\n\
    float4 matrix0, matrix1;\n\
    float4 theta1, theta2;\n\
    _viv_asm(COPY, theta1, theta_1, 16);\n\
    _viv_asm(COPY, theta2, theta_2, 16);\n\
\n\
    if (has_theta_1_1 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta1.x = data.x;\n\
    }\n\
\n\
    if (has_theta_1_2 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta1.y = data.x;\n\
    }\n\
\n\
    if (has_theta_1_3 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta1.z = data.x;\n\
    }\n\
\n\
    if (has_theta_2_1 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta2.x = data.x;\n\
    }\n\
\n\
    if (has_theta_2_2 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta2.y = data.x;\n\
    }\n\
\n\
    if (has_theta_2_3 == 0)\n\
    {\n\
        float4 data = read_imagef(input, coord);\n\
        coord.x ++;\n\
        theta2.z = data.x;\n\
    }\n\
\n\
    matrix0.x = theta2.y * scale.x;\n\
    matrix0.z = theta2.x * scale.z;\n\
    matrix1.x = ( theta2.z - theta2.y - theta2.x + 1) * i_width * 0.5f;\n\
    matrix0.y = theta1.y * scale.w;\n\
    matrix0.w = theta1.x * scale.y;\n\
    matrix1.y = ( theta1.z - theta1.y - theta1.x + 1) * i_height * 0.5f;\n\
    matrix1.zw = 2.0f * matrix0.xy;\n\
\n\
    coord.x = 0;\n\
    vxc_ushort8 dst;\n\
    _viv_asm(COPY, dst, matrix0, 16);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, matrix1, 16);\n\
    coord.x = 8;\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of get_matrix_vx*/

static const char group_normalization_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertEndInt16Fp32_4x4;\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform int output_ZP;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_F16(\n\
    image2d_array_t input, image2d_array_t output, float eps, int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0;\n\
        float sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_F16_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
    }\n\
\n\
    lcl_sum[lidx] = sumsqr.x;\n\
    lcl_sqr[lidx] = sumsqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, get_global_id(1), 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0;\n\
        float sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertEndInt16Fp32_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16toF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndInt16Fp32_4x4);\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16toU8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_uchar16 outval;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    float alpha = outputScale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertEndInt16Fp32_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16toU8_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_uchar16 outval;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    float alpha = outputScale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertEndInt16Fp32_4x4);\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of group_normalization_f16_vx*/

static const char group_normalization_f16_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertEndInt16Fp32_4x4;\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform int output_ZP;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16F32toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertEndInt16Fp32_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16F32toF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndInt16Fp32_4x4);\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16F32toU8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_uchar16 outval;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    float alpha = outputScale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertEndInt16Fp32_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_F16F32toU8_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, image2d_array_t output,\n\
    float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_uchar16 outval;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    float alpha = outputScale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertEndInt16Fp32_4x4);\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of group_normalization_f16_scale_vx*/

static const char group_normalization_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform float inFlScale_s2;\n\
_viv_uniform float input_fl_scale;\n\
_viv_uniform float inOut_fl_scale;\n\
_viv_uniform float output_fl_scale;\n\
\n\
_viv_uniform VXC_512Bits uniInt16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Secd_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toInt16_2x8;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_I16(\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 sumsqr = (vxc_float4)(0);\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniInt16SumSqr_dp8x2);\n\
            //tmpSumSqr += sumsqr;\n\
            tmpSumSqr.x += sumsqr.x;\n\
            sqr += (sumsqr.y * inFlScale_s2);\n\
        }\n\
        sum = tmpSumSqr.x * input_fl_scale;\n\
        //sqr = tmpSumSqr.y * inFlScale_s2;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_I16_2D(\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    vxc_short8 src0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 sumsqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniInt16SumSqr_dp8x2);\n\
        sqr = sumsqr.y * inFlScale_s2;\n\
        sum = sumsqr.x * input_fl_scale;\n\
        //sqr = tmpSumSqr.y * inFlScale_s2;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16toF16_2D(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16toI16(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toInt16_2x8);\n\
    VXC_WriteImage2DArray(output, coord, src2, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16toI16_2D(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toInt16_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of group_normalization_i16_vx*/

static const char group_normalization_i16_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform float input_fl_scale;\n\
_viv_uniform float inOut_fl_scale;\n\
_viv_uniform float output_fl_scale;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Secd_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toInt16_2x8;\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16F32toF16(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16F32toF16_2D(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16F32toI16(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_short8 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toInt16_2x8);\n\
    VXC_WriteImage2DArray(output, coord, src2, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I16F32toI16_2D(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int is2D,\n\
              float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_short8 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toInt16_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of group_normalization_i16_scale_vx*/

static const char group_normalization_i8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumInt8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSumInt8_16x1;\n\
_viv_uniform float inFlScale_s2;\n\
_viv_uniform float input_fl_scale;\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEndInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertTrdInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFthInt8Fp32_4x4;\n\
\n\
_viv_uniform float inOut_fl_scale;\n\
_viv_uniform float output_fl_scale;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_I8(\n\
    image2d_array_t input, image2d_array_t output, float eps, int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_char16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumInt8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSumInt8_16x1);\n\
            tmpSqr += (tmpSqr1);\n\
        }\n\
        sqr = tmpSqr * inFlScale_s2;\n\
        sum = tmpSum * input_fl_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_I8_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    vxc_char16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumInt8_16x1);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSumInt8_16x1);\n\
        sqr = tmpSqr1 * inFlScale_s2;\n\
        sum = tmpSum1 * input_fl_scale;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_char16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8toF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_char16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8toI8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_char16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, src2, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8toI8_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_char16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of group_normalization_i8_vx*/

static const char group_normalization_i8_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform float input_fl_scale;\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEndInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertTrdInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFthInt8Fp32_4x4;\n\
\n\
_viv_uniform float inOut_fl_scale;\n\
_viv_uniform float output_fl_scale;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8F32toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_char16 src0;\n\
    vxc_short8 outval;\n\
    vxc_half8 dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8F32toF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_char16 src0;\n\
    vxc_short8 outval;\n\
    vxc_half8 dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8F32toI8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_char16 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, src2, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_I8F32toI8_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_char16 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of group_normalization_i8_scale_vx*/

static const char group_normalization_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform float scale_inOut;\n\
_viv_uniform float outputScale;\n\
_viv_uniform int output_ZP;\n\
\n\
_viv_uniform VXC_512Bits uniResetFp32_4x4;\n\
_viv_uniform int group_stride;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_U8(\n\
    image2d_array_t input, image2d_array_t output, float eps, int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1 = 0, tmpSqr1 = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1);\n\
        }\n\
        sqr += (tmpSqr * e2InScale + rowSumScale);\n\
        sum = (tmpSum + sumInZp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_sumsqr_U8_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int is2D)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSqr, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    if(gidx < width)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
        tmpSqr = tmpSqr1 + tmpZp1 * tmpSum1;\n\
        sqr = (tmpSqr * e2InScale + rowSumScale);\n\
        sum = (tmpSum1 + sumInZp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, get_global_id(1), 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_meanvari(\n\
    image2d_t input, image2d_t output, float eps, float group_ratio)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_uchar16 src0;\n\
    float2 sum_sqr = (float2)(0);\n\
    vxc_float4 mean_vari;\n\
    VXC_DP4x4(mean_vari, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniResetFp32_4x4);\n\
\n\
    __local float2 lcl_data[16];\n\
    __local float2 lcl_sum[4];\n\
\n\
    for(; coord.x < group_stride; coord.x += 64)\n\
    {\n\
        mean_vari += read_imagef(input, coord);\n\
    }\n\
    lcl_data[lidx] = mean_vari.xy;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx < 4)\n\
    {\n\
        float2 tmpSum = (float2)(0);\n\
        for(int i = lidx; i < 16; i+=4)\n\
        {\n\
            tmpSum += lcl_data[i];\n\
        }\n\
        lcl_sum[lidx] = tmpSum;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx == 0)\n\
    {\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum_sqr += lcl_sum[i];\n\
        }\n\
        mean_vari.xy = sum_sqr * group_ratio;\n\
        mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
        mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
        coord.x = 0;\n\
        write_imagef(output, coord, mean_vari);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8toU8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_uchar16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, src2, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8toU8_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_uchar16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8F32toU8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_uchar16 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, src2, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8F32toU8_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_uchar16 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of group_normalization_u8_vx*/

static const char group_normalization_u8_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_uchar16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8toF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_uchar16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8F32toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 0);\n\
    vxc_uchar16 src0;\n\
    vxc_short8 outval;\n\
    vxc_half8 dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage2DArray(output, coord, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void group_norm_U8F32toF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int is2D, float rSpaceOrg, int pStride)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 0);\n\
    vxc_uchar16 src0;\n\
    vxc_short8 outval;\n\
    vxc_half8 dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f;\n\
\n\
    vxc_float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    bias_f = read_imagef(bias, coord_para.xy);\n\
    scale_f = read_imagef(scale, coord_para.xy);\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of group_normalization_u8_f16_vx*/

static const char grucell_activation_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE     (1.44269502f)\n\
#define twoLogE  (2.88539004f)\n\
\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hsigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvDatatoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float4 tensorScale;\n\
_viv_uniform float4 tensorZP;\n\
\n\
#define GRUCELL_ACTIVATION_SIGMOID_TANH(name0, name1, name2, name3, activater, \\\n\
        type00, type01, type10, type11, type20, type21, dst_type, conv_type, copy_type) \\\n\
__kernel void grucell_activation_##name0##_##name1##_##name2##_to_##name3##_##activater \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __read_only  image2d_array_t input2, \\\n\
    __write_only image2d_array_t output, \\\n\
    __write_only image2d_array_t hstate, \\\n\
                             int gate_activation, \\\n\
                             int candidate_activation \\\n\
    ) \\\n\
{ \\\n\
    type00 src00; \\\n\
    type01 src01; \\\n\
    type00 src10; \\\n\
    type01 src11; \\\n\
    type00 src20; \\\n\
    type01 src21; \\\n\
 \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    VXC_ReadImage(src00, input0, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src01, src00, 8); \\\n\
    VXC_ReadImage(src10, input1, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, src10, 8); \\\n\
    VXC_ReadImage(src20, input2, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src21, src20, 8); \\\n\
 \\\n\
    float4 zt, ht, ht_1; \\\n\
    VXC_DP4x4(zt, src01, src01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4); \\\n\
    VXC_DP4x4(ht, src11, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4); \\\n\
    VXC_DP4x4(ht_1, src21, src21, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4); \\\n\
 \\\n\
    zt = zt * tensorScale.xxxx - tensorZP.xxxx; \\\n\
    zt = activater(zt); \\\n\
 \\\n\
    ht = ht * tensorScale.yyyy - tensorZP.yyyy; \\\n\
    ht = tangentH(ht); \\\n\
 \\\n\
    ht_1 = ht_1 * tensorScale.zzzz - tensorZP.zzzz; \\\n\
 \\\n\
    ht = ht - zt * ht; \\\n\
    ht = zt * ht_1 + ht; \\\n\
 \\\n\
    ht = ht  * tensorScale.wwww + tensorZP.wwww; \\\n\
    conv_type dst0; \\\n\
    dst_type dst1; \\\n\
    copy_type dst; \\\n\
 \\\n\
    _viv_asm(CONV_RTE, dst0, ht); \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
#define UCHAR8  vxc_uchar8\n\
#define SHORT8  vxc_short8\n\
#define HALF8   vxc_half8\n\
\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(U8,  U8,  U8,  U8, sigmoid,\n\
                                UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, int4,  UCHAR8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, F16, sigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  HALF8,  half4, SHORT8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, U8, sigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  UCHAR8, int4,  UCHAR8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(U8,  U8,  U8,  U8, hsigmoid,\n\
                                UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, UCHAR8, int4,  UCHAR8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, F16, hsigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  HALF8,  half4, SHORT8)\n\
GRUCELL_ACTIVATION_SIGMOID_TANH(F16, F16, F16, U8, hsigmoid,\n\
                                SHORT8, HALF8,  SHORT8, HALF8,  SHORT8, HALF8,  UCHAR8, int4,  UCHAR8)\n\
\n\
#undef UCHAR8\n\
#undef SHORT8\n\
#undef HALF8\n\
"; /* end of grucell_activation_vx*/

static const char grucell_activation_sma_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniA_Minus_B_2x8;\n\
_viv_uniform VXC_512Bits uniA_Times_B_2x8;\n\
_viv_uniform VXC_512Bits uniA_Plus_B_2x8;\n\
__kernel void grucell_activation_sma_F16_F16_F16toF16\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  h_status\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_half8   src0, src1, src2, minus, dst;\n\
    vxc_ushort8 vec0, vec1, vec2;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage2DArray(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(minus, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Minus_B_2x8);\n\
    VXC_DP2x8(dst, minus, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(h_status, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_sma_F16_F16_F16toF16_2D\n\
    (\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_array_t   input1,\n\
    __read_only image2d_array_t   input2,\n\
    __write_only image2d_array_t  output,\n\
    __write_only image2d_array_t  h_status\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_half8   src0, src1, src2, minus, dst;\n\
    vxc_ushort8 vec0, vec1, vec2;\n\
\n\
    VXC_ReadImage(vec0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
    VXC_ReadImage(vec2, input2, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src2, vec2, 16);\n\
\n\
    VXC_DP2x8(minus, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Minus_B_2x8);\n\
    VXC_DP2x8(dst, minus, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Times_B_2x8);\n\
    VXC_DP2x8(dst, dst, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniA_Plus_B_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(h_status, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of grucell_activation_sma_vx*/

static const char grucell_activation_z_h_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float4 sigmoid_func(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1.0f + exp2(x);\n\
    return 1.0f / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tanh_func(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1.0f / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniF16PlusF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniF16PlusF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define GRUCELL_F16_F16TOF16(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_F16_F16toF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, v3, 16); \\\n\
 \\\n\
    float4 h; \\\n\
    VXC_DP4x4(h, src2, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = act_func(z); \\\n\
    h = tanh_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half4 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_F16_F16TOF16(SIGMOID, sigmoid_func)\n\
\n\
_viv_uniform float hstate_in_scale;\n\
_viv_uniform float hstate_in_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
#define GRUCELL_QNT_F16TO_QNT(name0, name1, act_name, act_func, src0_type, dst_type) \\\n\
__kernel void grucell_activation_z_h_##name0##_F16to##name1##_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type src3; \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src4, src5, src6; \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(src3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 h; \\\n\
    VXC_DP4x4(h, src2, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = act_func(z); \\\n\
    h = tanh_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    h_tm = h_tm * hstate_in_scale + hstate_in_tail; \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    result = result * output_scale + output_zp; \\\n\
    int4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    dst_type dst; \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_QNT_F16TO_QNT(U8,  U8,  SIGMOID, sigmoid_func, vxc_uchar8, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_QNT(I8,  I8,  SIGMOID, sigmoid_func, vxc_char8,  vxc_char8)\n\
GRUCELL_QNT_F16TO_QNT(I16, I16, SIGMOID, sigmoid_func, vxc_short8, vxc_short8)\n\
"; /* end of grucell_activation_z_h_vx*/

static const char grucell_cdnn_activation_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE     (1.44269502f)\n\
#define twoLogE  (2.88539004f)\n\
\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvDatatoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uiF16AddF16_4x4;\n\
\n\
__kernel void grucell_activation_cdnn_sep_F16_F16_F16_to_F16_NC\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord);\n\
    r3 = read_imagef(cond_r, coord);\n\
\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord);\n\
    z3 = read_imagef(cond_z, coord);\n\
\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord);\n\
    c3 = read_imagef(cond_c, coord);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2 + r3;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2 + z3;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_F16_F16_F16_to_F16_CN\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2.xxxx + r3.xxxx;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(s7, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_F16_F16_F16_to_F16_CN_FULL\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(s2, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(s4, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2.xxxx + r3.xxxx;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
\n\
__kernel void grucell_activation_cdnn_F16_F16_F16_to_F16\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_rzc,\n\
    __read_only  image2d_array_t recur_rzc,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_ushort8 s0, s1;\n\
    vxc_half8   r0, r1;\n\
    vxc_ushort8 s2, s3;\n\
    vxc_half8   z0, z1;\n\
    vxc_ushort8 s4, s5;\n\
    vxc_half8   c0, c1;\n\
    float4      r, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(1) * 3, get_global_id(1));\n\
\n\
    VXC_ReadImage(s0, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r0, s0, 8);\n\
    VXC_ReadImage(s1, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, r1, s1, 8);\n\
    r2 = read_imagef(bias_r, coord.xy);\n\
    r3 = read_imagef(cond_r, coord.xy);\n\
\n\
    VXC_ReadImage(s2, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z0, s2, 8);\n\
    VXC_ReadImage(s3, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, z1, s3, 8);\n\
    z2 = read_imagef(bias_z, coord.xy);\n\
    z3 = read_imagef(cond_z, coord.xy);\n\
\n\
    VXC_ReadImage(s4, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, s4, 8);\n\
    VXC_ReadImage(s5, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c1, s5, 8);\n\
    c2 = read_imagef(bias_c, coord.xy);\n\
    c3 = read_imagef(cond_c, coord.xy);\n\
\n\
    VXC_DP4x4(r, r0, r1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    r = r + r2 + r3;\n\
    VXC_DP4x4(z, z0, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uiF16AddF16_4x4);\n\
    z = z + z2 + z3;\n\
\n\
    vxc_ushort8 s7;\n\
    vxc_half8 h;\n\
    VXC_ReadImage(s7, prev_state, coord.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, h, s7, 8);\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
\n\
    state = z * (state - c) + c;\n\
\n\
    half4 dst0;\n\
    vxc_half4 dst1;\n\
    vxc_short4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    _viv_asm(COPY, dst, dst1, 8);\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of grucell_cdnn_activation_vx*/

static const char grucell_cdnn_activation_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE     (1.44269502f)\n\
#define twoLogE  (2.88539004f)\n\
\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvDatatoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float input_r_scale;\n\
_viv_uniform float input_r_tail;\n\
_viv_uniform float recur_r_scale;\n\
_viv_uniform float recur_r_tail;\n\
_viv_uniform float input_z_scale;\n\
_viv_uniform float input_z_tail;\n\
_viv_uniform float recur_z_scale;\n\
_viv_uniform float recur_z_tail;\n\
_viv_uniform float input_c_scale;\n\
_viv_uniform float input_c_tail;\n\
_viv_uniform float recur_c_scale;\n\
_viv_uniform float recur_c_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
__kernel void grucell_activation_cdnn_sep_U8_U8_U8_to_U8_NC\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8  r00, r01;\n\
    vxc_uchar8  z0, z1;\n\
    vxc_uchar8  c0, c1;\n\
    float4      r, r0, r1, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord);\n\
    r3 = read_imagef(cond_r, coord);\n\
\n\
    VXC_ReadImage(z0, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord);\n\
    z3 = read_imagef(cond_z, coord);\n\
\n\
    VXC_ReadImage(c0, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord);\n\
    c3 = read_imagef(cond_c, coord);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2 + r3;\n\
\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_z_scale + input_z_tail;\n\
    r1 = r1 * recur_z_scale + recur_z_tail;\n\
    z = r0 + r1 + z2 + z3;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_U8_U8_U8_to_U8_CN\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8   r00, r01;\n\
    vxc_uchar8   z0, z1;\n\
    vxc_uchar8   c0, c1;\n\
    float4      r, r2, r3, r0, r1;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(z0, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(c0, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2.xxxx + r3.xxxx;\n\
\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_z_scale + input_z_tail;\n\
    r1 = r1 * recur_z_scale + recur_z_tail;\n\
    z = r0 + r1 + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord.yx, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(h, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(h, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(h, prev_state, coord.yx, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x ++;\n\
    VXC_WriteImage(output, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.yx, dst, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_sep_U8_U8_U8_to_U8_CN_FULL\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_r,\n\
    __read_only  image2d_array_t input_z,\n\
    __read_only  image2d_array_t input_c,\n\
    __read_only  image2d_array_t recur_r,\n\
    __read_only  image2d_array_t recur_z,\n\
    __read_only  image2d_array_t recur_c,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8   r00, r01;\n\
    vxc_uchar8   z0, z1;\n\
    vxc_uchar8   c0, c1;\n\
    float4      r, r2, r3, r0, r1;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_r, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.yx);\n\
    r3 = read_imagef(cond_r, coord.yx);\n\
\n\
    VXC_ReadImage(z0, input_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_z, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord.yx);\n\
    z3 = read_imagef(cond_z, coord.yx);\n\
\n\
    VXC_ReadImage(c0, input_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_c, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord.yx);\n\
    c3 = read_imagef(cond_c, coord.yx);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2.xxxx + r3.xxxx;\n\
\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_z_scale + input_z_tail;\n\
    r1 = r1 * recur_z_scale + recur_z_tail;\n\
    z = r0 + r1 + z2.xxxx + z3.xxxx;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2.xxxx * r + c3.xxxx;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void grucell_activation_cdnn_U8_U8_U8_to_U8\n\
    (\n\
    __read_only  image2d_array_t prev_state,\n\
    __read_only  image2d_array_t input_rzc,\n\
    __read_only  image2d_array_t recur_rzc,\n\
    __read_only  image2d_t       bias_r,\n\
    __read_only  image2d_t       bias_z,\n\
    __read_only  image2d_t       bias_c,\n\
    __read_only  image2d_t       cond_r,\n\
    __read_only  image2d_t       cond_z,\n\
    __read_only  image2d_t       cond_c,\n\
    __write_only image2d_array_t output,\n\
    __write_only image2d_array_t hstate,\n\
                             int gate_activation,\n\
                             int candidate_activation,\n\
                             int batch_first\n\
    )\n\
{\n\
    vxc_uchar8   r00, r01;\n\
    vxc_uchar8   z0, z1;\n\
    vxc_uchar8   c0, c1;\n\
    float4      r, r0, r1, r2, r3;\n\
    float4      z, z2, z3;\n\
    float4      c, c2, c3;\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(1) * 3, get_global_id(1));\n\
\n\
    VXC_ReadImage(r00, input_rzc, coord.xz, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(r01, recur_rzc, coord.xz, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    r2 = read_imagef(bias_r, coord.xy);\n\
    r3 = read_imagef(cond_r, coord.xy);\n\
\n\
    VXC_ReadImage(z0, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(z1, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    z2 = read_imagef(bias_z, coord.xy);\n\
    z3 = read_imagef(cond_z, coord.xy);\n\
\n\
    VXC_ReadImage(c0, input_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(c1, recur_rzc, coord.xz, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    c2 = read_imagef(bias_c, coord.xy);\n\
    c3 = read_imagef(cond_c, coord.xy);\n\
\n\
    VXC_DP4x4(r0, r00, r00, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, r01, r01, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * recur_r_scale + recur_r_tail;\n\
    r = r0 + r1 + r2 + r3;\n\
    VXC_DP4x4(r0, z0, z0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(r1, z1, z1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    r0 = r0 * input_r_scale + input_r_tail;\n\
    r1 = r1 * input_r_scale + recur_r_tail;\n\
    z = r0 + r1 + z2 + z3;\n\
\n\
    vxc_uchar8 h;\n\
    VXC_ReadImage(h, prev_state, coord.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    r = sigmoid(r);\n\
    z = sigmoid(z);\n\
\n\
    c = c2 * r + c3;\n\
    VXC_DP4x4(c2, c0, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    VXC_DP4x4(c3, c1, c1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    c2 = c2 * input_c_scale + input_c_tail;\n\
    c3 = c3 * recur_c_scale + recur_c_tail;\n\
    c = c2 + c3 * r + c;\n\
    c = tangentH(c);\n\
\n\
    float4 state;\n\
    VXC_DP4x4(state, h, h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvDatatoFp32_4x4);\n\
    state = state * input_scale + input_tail;\n\
    state = z * (state - c) + c;\n\
\n\
    state = state * output_scale + output_zp;\n\
\n\
    int4 dst0;\n\
    vxc_uchar4 dst;\n\
    _viv_asm(CONV_RTE, dst0, state);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8);\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(hstate, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of grucell_cdnn_activation_u8_vx*/

static const char grucell_h_times_activation_r_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float4 sigmoid_func(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1.0f + exp2(x);\n\
    return 1.0f / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniF16PlusF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniF16PlusF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define GRUCELL_F16_F16TOF16(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_F16_F16toF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __write_only image2d_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, v3, 16); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = act_func(r); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 result = r * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half4 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_F16_F16TOF16(SIGMOID, sigmoid_func)\n\
\n\
_viv_uniform float hstate_in_scale;\n\
_viv_uniform float hstate_in_tail;\n\
#define GRUCELL_QNT_F16TO_F16(name0, act_name, act_func, src0_type) \\\n\
__kernel void grucell_h_times_activation_r_##name0##_F16toF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __write_only image2d_t output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type src3; \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(src3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = act_func(r); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    h_tm = h_tm * hstate_in_scale + hstate_in_tail; \\\n\
    float4 result = r * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half8 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_QNT_F16TO_F16(U8,  SIGMOID, sigmoid_func, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_F16(I8,  SIGMOID, sigmoid_func, vxc_char8)\n\
GRUCELL_QNT_F16TO_F16(I16, SIGMOID, sigmoid_func, vxc_short8)\n\
"; /* end of grucell_h_times_activation_r_vx*/

static const char grucell_reset_after_activation_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float4 sigmoid_func(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1.0f + exp2(x);\n\
    return 1.0f / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tanh_func(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1.0f / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniF16PlusF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniF16PlusF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertF16_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define GRUCELL_F16_F16TOF16(act_name, act_func) \\\n\
__kernel void grucell_reset_after_activation_F16_F16toF16_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src3, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(v3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, v3, 16); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = act_func(r); \\\n\
    float4 h0, h1; \\\n\
    VXC_DP4x4(h1, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    VXC_DP4x4(h0, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 h = h0 + r * h1; \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = act_func(z); \\\n\
    h = tanh_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    half4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    vxc_half4 dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    vxc_short4 dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_F16_F16TOF16(SIGMOID, sigmoid_func)\n\
\n\
_viv_uniform float hstate_in_scale;\n\
_viv_uniform float hstate_in_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
#define GRUCELL_QNT_F16TO_QNT(name0, name1, act_name, act_func, src0_type, dst_type) \\\n\
__kernel void grucell_reset_after_activation_##name0##_F16to##name1##_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_z_conv, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t input_h_conv, \\\n\
    __read_only  image2d_t hstate_z_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __read_only  image2d_t hstate_h_conv, \\\n\
    __write_only image2d_t output, \\\n\
    __write_only image2d_t hstate_out \\\n\
    ) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type src3; \\\n\
    vxc_short8 v0, v1, v2, v3, v4, v5, v6; \\\n\
    vxc_half8 src0, src1, src2, src4, src5, src6; \\\n\
    VXC_ReadImage(v0, input_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, v0, 16); \\\n\
    VXC_ReadImage(v1, hstate_r_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, v1, 16); \\\n\
    VXC_ReadImage(v2, hstate_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, v2, 16); \\\n\
    VXC_ReadImage(v4, input_h_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, v4, 16); \\\n\
    VXC_ReadImage(v5, input_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src5, v5, 16); \\\n\
    VXC_ReadImage(v6, hstate_z_conv, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src6, v6, 16); \\\n\
    VXC_ReadImage(src3, hstate_in, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 r; \\\n\
    VXC_DP4x4(r, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    r = act_func(r); \\\n\
    float4 h0, h1; \\\n\
    VXC_DP4x4(h1, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    VXC_DP4x4(h0, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    float4 h = h0 + r * h1; \\\n\
    float4 z; \\\n\
    VXC_DP4x4(z, src5, src6, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniF16PlusF16_0_4x4); \\\n\
    z = act_func(z); \\\n\
    h = tanh_func(h); \\\n\
    float4 h_tm; \\\n\
    VXC_DP4x4(h_tm, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertF16_0_4x4); \\\n\
    h_tm = h_tm * hstate_in_scale + hstate_in_tail; \\\n\
    float4 result = (1 - z) * h + z * h_tm; \\\n\
    result = result * output_scale + output_zp; \\\n\
    int4 dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    dst_type dst; \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage(output, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(hstate_out, coord_in, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
GRUCELL_QNT_F16TO_QNT(U8,  U8,  SIGMOID, sigmoid_func, vxc_uchar8, vxc_uchar8)\n\
GRUCELL_QNT_F16TO_QNT(I8,  I8,  SIGMOID, sigmoid_func, vxc_char8,  vxc_char8)\n\
GRUCELL_QNT_F16TO_QNT(I16, I16, SIGMOID, sigmoid_func, vxc_short8, vxc_short8)\n\
"; /* end of grucell_reset_after_activation_vx*/

static const char hswish_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define HSWISH_PROCESS(read_fun, write_fun, src_type, src_copy_type, convert_type, dst_type, dst_copy_type, \\\n\
                     INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
    float4 vecA, vecB, vecC, vecD, vecE, vecDstA, vecDstB; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * INSCALE + INTAIL; \\\n\
    vecB = vecB * INSCALE + INTAIL; \\\n\
    vecC = vecA + 3.0f; \\\n\
    vecD = vecB + 3.0f; \\\n\
    vecE = 6.0f; \\\n\
    _viv_asm(CLAMP0MAX, vecDstA, vecC, vecE); \\\n\
    _viv_asm(CLAMP0MAX, vecDstB, vecD, vecE); \\\n\
    vecA = vecA * vecDstA; \\\n\
    vecB = vecB * vecDstB; \\\n\
    vecA = vecA / 6.0f; \\\n\
    vecB = vecB / 6.0f; \\\n\
    vecA = vecA * OUTSCALE + OUTZP; \\\n\
    vecB = vecB * OUTSCALE + OUTZP; \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define HSWISH_FUNC(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, \\\n\
                   dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void hswish_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    HSWISH_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, src_type, \\\n\
                 src_copy_type, convert_type, dst_type, dst_copy_type, \\\n\
                 INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
HSWISH_FUNC(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
HSWISH_FUNC(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
HSWISH_FUNC(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
HSWISH_FUNC(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
HSWISH_FUNC(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
HSWISH_FUNC(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, \\\n\
            inputScale, inputTail, outputScale, outputZP)\n\
HSWISH_FUNC(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
HSWISH_FUNC(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
#define HSWISH_FUNC_2D(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, \\\n\
                      dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void hswish_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    HSWISH_PROCESS(VXC_ReadImage, VXC_WriteImage, src_type, src_copy_type, convert_type, dst_type, \\\n\
                   dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
HSWISH_FUNC_2D(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
HSWISH_FUNC_2D(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
HSWISH_FUNC_2D(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
HSWISH_FUNC_2D(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, inputScale, \\\n\
               inputTail, outputScale, outputZP)\n\
HSWISH_FUNC_2D(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
HSWISH_FUNC_2D(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
HSWISH_FUNC_2D(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define HSWISH_BF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    float4 vecA, vecB, vecC, vecD, vecE, vecDstA, vecDstB; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecC = vecA + 3.0f; \\\n\
    vecD = vecB + 3.0f; \\\n\
    vecE = 6.0f; \\\n\
    _viv_asm(CLAMP0MAX, vecDstA, vecC, vecE); \\\n\
    _viv_asm(CLAMP0MAX, vecDstB, vecD, vecE); \\\n\
    vecA = vecA * vecDstA; \\\n\
    vecB = vecB * vecDstB; \\\n\
    vecA = vecA / 6.0f; \\\n\
    vecB = vecB / 6.0f; \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void hswish_BF16toBF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_BF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray);\n\
}\n\
\n\
__kernel void hswish_BF16toBF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_BF16_PROCESS(VXC_ReadImage, VXC_WriteImage);\n\
}\n\
"; /* end of hswish_vx*/

static const char instance_normalization_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertEndInt16Fp32_4x4;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_F16(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0;\n\
        float sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_F16_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0;\n\
        float sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_F16toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
    bias_f = read_imagef(bias, coord_para);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
\n\
    coord_in.y ++;\n\
\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndInt16Fp32_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                    VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_F16toF16_2D(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int4 coord = (int4)(get_global_id(0), gidy, 0, 0);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    int endH = gidy + height;\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
    bias_f = read_imagef(bias, coord_para);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndInt16Fp32_4x4);\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of instance_normalization_f16_vx*/

static const char instance_normalization_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform float inFlScale_s2;\n\
_viv_uniform float input_fl_scale;\n\
_viv_uniform float inOut_fl_scale;\n\
_viv_uniform float output_fl_scale;\n\
\n\
_viv_uniform VXC_512Bits uniInt16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Secd_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toInt16_2x8;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_I16(\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz);\n\
    vxc_short8 src0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 sumsqr = (vxc_float4)(0);\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, 0, \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniInt16SumSqr_dp8x2);\n\
            //tmpSumSqr += sumsqr;\n\
            tmpSumSqr.x += sumsqr.x;\n\
            sqr += (sumsqr.y * inFlScale_s2);\n\
        }\n\
        sum = tmpSumSqr.x * input_fl_scale;\n\
        //sqr = tmpSumSqr.y * inFlScale_s2;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_I16_2D(\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_short8 src0;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 sumsqr = (vxc_float4)(0);\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, 0,\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniInt16SumSqr_dp8x2);\n\
            //tmpSumSqr += sumsqr;\n\
            tmpSumSqr.x += sumsqr.x;\n\
            sqr += (sumsqr.y * inFlScale_s2);\n\
        }\n\
        sum = tmpSumSqr.x * input_fl_scale;\n\
        //sqr = tmpSumSqr.y * inFlScale_s2;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            //sum += lcl_sum[i];\n\
            //sqr += lcl_sqr[i];\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t bias,\n\
    image2d_array_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0,\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, 0, \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y ++;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                    VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I16toF16_2D(\n\
    image2d_array_t input,\n\
    image2d_array_t bias,\n\
    image2d_array_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int4 coord = (int4)(get_global_id(0), gidy, 0, 0);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    int endH = gidy + height;\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0,\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord.xy, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I16toI16(\n\
    image2d_array_t input,\n\
    image2d_array_t bias,\n\
    image2d_array_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    vxc_short8 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0,\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, 0, \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y ++;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toInt16_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, src2, \\\n\
                    VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I16toI16_2D(\n\
    image2d_array_t input,\n\
    image2d_array_t bias,\n\
    image2d_array_t scale,\n\
    image2d_t meanVari,\n\
    image2d_array_t output,\n\
              float eps,\n\
              int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    int endH = gidy + height;\n\
    vxc_short8 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0,\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord, 0,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toInt16_2x8);\n\
    VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of instance_normalization_i16_vx*/

static const char instance_normalization_i8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniSumInt8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSumInt8_16x1;\n\
_viv_uniform float inFlScale_s2;\n\
_viv_uniform float input_fl_scale;\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEndInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertTrdInt8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFthInt8Fp32_4x4;\n\
\n\
_viv_uniform float inOut_fl_scale;\n\
_viv_uniform float output_fl_scale;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_I8(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz);\n\
    vxc_char16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumInt8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSumInt8_16x1);\n\
            tmpSqr += (tmpSqr1);\n\
        }\n\
        sqr = tmpSqr * inFlScale_s2;\n\
        sum = tmpSum * input_fl_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_I8_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_char16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumInt8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSumInt8_16x1);\n\
            tmpSqr += (tmpSqr1);\n\
        }\n\
        sqr = tmpSqr * inFlScale_s2;\n\
        sum = tmpSum * input_fl_scale;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I8toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    vxc_char16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
    bias_f = read_imagef(bias, coord_para);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    coord_para = coord;\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_para.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height;)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_para.xy = coord.xy;\n\
    coord.y++;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_para, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    coord_para.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_para, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I8toF16_2D(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int4 coord = (int4)(get_global_id(0), gidy, 0, 0);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    int endH = gidy + height;\n\
    vxc_char16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    for(; coord.y < endH;)\n\
    {\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_para = coord;\n\
    coord.y++;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord_para.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_para.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord_para.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I8toI8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    vxc_char16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y ++;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I8toI8_2D(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    int endH = gidy + height;\n\
    vxc_char16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertTrdInt8Fp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFthInt8Fp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of instance_normalization_i8_vx*/

static const char instance_normalization_scale_f32_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float scale_inOut;\n\
_viv_uniform float outputScale;\n\
_viv_uniform int output_ZP;\n\
_viv_uniform float inOut_fl_scale;\n\
_viv_uniform float output_fl_scale;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Fst_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt16Fp32Secd_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toInt16_2x8;\n\
\n\
#define INSTANCENORM_8BITS_F32(src1_type_name, read_type) \\\n\
__kernel void instance_norm_##src1_type_name##F32to##src1_type_name( \\\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps, int rsFlg) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz); \\\n\
    int2 coord_para = (int2)(gidz, 0); \\\n\
    read_type src0, src2; \\\n\
    float scale_vari, bias_val; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    Image img3 = create_image_from_image2d(meanVari, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scal_ptr = (__global float*)img2.ptr; \\\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, coord_para.yx); \\\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr; \\\n\
 \\\n\
    float bval = bias_ptr[gidz]; \\\n\
    float sval = scal_ptr[gidz]; \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += vari_ptr[i]; \\\n\
    } \\\n\
    mean_vari *= dimRatio; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    scale_vari = sval * mean_vari.s1; \\\n\
    short zp = inputZP; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    float alpha = scale_inOut * scale_vari; \\\n\
    bias_val = (bval - scale_vari * mean_vari.s0) * outputScale + output_ZP; \\\n\
 \\\n\
    int8 input_desc, output_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr); \\\n\
 \\\n\
    for(coord.y = 0; coord.y < height; coord.y++) \\\n\
    { \\\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in.y ++; \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert2ndUint8SubZpToFp32_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert3rdUint8SubZpToFp32_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert4thUint8SubZpToFp32_4x4); \\\n\
    norm = tmpData0 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    norm = tmpData2 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCENORM_8BITS_F32(U8, vxc_uchar16)\n\
INSTANCENORM_8BITS_F32(I8, vxc_char16)\n\
\n\
#define INSTANCENORM_8BITS_F32_2D(src1_type_name, read_type) \\\n\
__kernel void instance_norm_##src1_type_name##F32to##src1_type_name##_2D( \\\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari, \\\n\
    image2d_array_t output, float eps, int rsFlg) \\\n\
{ \\\n\
    int gidz = get_global_id(1); \\\n\
    int gidy = gidz * height; \\\n\
    int2 coord = (int2)(get_global_id(0), gidy); \\\n\
    int2 coord_para = (int2)(gidz, 0); \\\n\
    int endH = gidy + height; \\\n\
    read_type src0, src2; \\\n\
    float scale_vari, bias_val; \\\n\
    vxc_float4 mean_vari = (vxc_float4)(0); \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(bias, 4); \\\n\
    Image img2 = create_image_from_image2d(scale, 4); \\\n\
    Image img3 = create_image_from_image2d(meanVari, 4); \\\n\
    __global float* bias_ptr = (__global float*)img1.ptr; \\\n\
    __global float* scal_ptr = (__global float*)img2.ptr; \\\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, coord_para.yx); \\\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr; \\\n\
 \\\n\
    float bval = bias_ptr[gidz]; \\\n\
    float sval = scal_ptr[gidz]; \\\n\
 \\\n\
    for(int i = 0; i < group_num; i++) \\\n\
    { \\\n\
        mean_vari += vari_ptr[i]; \\\n\
    } \\\n\
 \\\n\
    mean_vari *= dimRatio; \\\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps; \\\n\
    mean_vari.s1 = rsqrt(mean_vari.s1); \\\n\
 \\\n\
    scale_vari = sval * mean_vari.s1; \\\n\
    short zp = inputZP; \\\n\
    vxc_int4 tmpVal0, tmpVal1; \\\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm; \\\n\
    float alpha = scale_inOut * scale_vari; \\\n\
    bias_val = (bval - scale_vari * mean_vari.s0) * outputScale + output_ZP; \\\n\
 \\\n\
    for(; coord.y < endH; coord.y++) \\\n\
    { \\\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert2ndUint8SubZpToFp32_4x4); \\\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert3rdUint8SubZpToFp32_4x4); \\\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
             uniConvert4thUint8SubZpToFp32_4x4); \\\n\
    norm = tmpData0 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData1 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    norm = tmpData2 * alpha + bias_val; \\\n\
    tmpVal0 = convert_int4_rte(norm); \\\n\
    norm = tmpData3 * alpha + bias_val; \\\n\
    tmpVal1 = convert_int4_rte(norm); \\\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
INSTANCENORM_8BITS_F32_2D(U8, vxc_uchar16)\n\
INSTANCENORM_8BITS_F32_2D(I8, vxc_char16)\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I16F32toI16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_para = (int2)(gidz, 0);\n\
    vxc_short8 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4  mean_vari = (vxc_float4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    Image img3 = create_image_from_image2d(meanVari, 4);\n\
    __global float* bias_ptr = (__global float*)img1.ptr;\n\
    __global float* scal_ptr = (__global float*)img2.ptr;\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, coord_para.yx);\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr;\n\
\n\
    float bval = bias_ptr[gidz];\n\
    float sval = scal_ptr[gidz];\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += vari_ptr[i];\n\
    }\n\
\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = sval * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bval - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y ++;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toInt16_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, src2, \\\n\
                    VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_I16F32toI16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t meanVari, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(gidz, 0);\n\
    int endH = gidy + height;\n\
    vxc_short8 src0, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    Image img3 = create_image_from_image2d(meanVari, 4);\n\
    __global float* bias_ptr = (__global float*)img1.ptr;\n\
    __global float* scal_ptr = (__global float*)img2.ptr;\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, coord_para.yx);\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr;\n\
\n\
    float bval = bias_ptr[gidz];\n\
    float sval = scal_ptr[gidz];\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += vari_ptr[i];\n\
    }\n\
\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = sval * mean_vari.s1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    float alpha = inOut_fl_scale * scale_vari;\n\
    bias_val = (bval - scale_vari * mean_vari.s0) * output_fl_scale;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Fst_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt16Fp32Secd_4x4);\n\
    vxc_float4 norm;\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toInt16_2x8);\n\
    VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of instance_normalization_scale_f32_vx*/

static const char instance_normalization_scale_f32_bf16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
constant vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
constant float4 one = (float4)(1.0, 1.0, 1.0, 1.0);\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_BF16(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz);\n\
    vxc_short8 src0, src1, src2;\n\
    float4 srcA, srcB;\n\
    vxc_float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part0_2x8);\n\
            VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, srcA, src1, 16);\n\
            _viv_asm(COPY, srcB, src2, 16);\n\
            sum += dot(srcA, one) + dot(srcB, one);\n\
            sqr += dot(srcA * srcA, one) + dot(srcB * srcB, one);\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0;\n\
        sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_BF16_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_short8 src0, src1, src2;\n\
    float4 srcA, srcB;\n\
    vxc_float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part0_2x8);\n\
            VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, srcA, src1, 16);\n\
            _viv_asm(COPY, srcB, src2, 16);\n\
            sum += dot(srcA, one) + dot(srcB, one);\n\
            sqr += dot(srcA * srcA, one) + dot(srcB * srcB, one);\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0;\n\
        sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_BF16F32toBF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    vxc_short8 src0, src1, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    Image img3 = create_image_from_image2d(meanVari, 4);\n\
    __global float* bias_ptr = (__global float*)img1.ptr;\n\
    __global float* scal_ptr = (__global float*)img2.ptr;\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, (int2)(0, gidz));\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr;\n\
\n\
    float bval = bias_ptr[gidz];\n\
    float sval = scal_ptr[gidz];\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += vari_ptr[i];\n\
    }\n\
\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = sval * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    bias_val = (bval - scale_vari * mean_vari.s0);\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y ++;\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, tmpData0, src1, 16);\n\
    _viv_asm(COPY, tmpData1, src2, 16);\n\
\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(COPY, src0, norm, 16);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(COPY, src1, norm, 16);\n\
    VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, src2, \\\n\
                    VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_BF16F32toBF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(gidz, 0);\n\
    int endH = gidy + height;\n\
    vxc_short8 src0, src1, src2;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    Image img3 = create_image_from_image2d(meanVari, 4);\n\
    __global float* bias_ptr = (__global float*)img1.ptr;\n\
    __global float* scal_ptr = (__global float*)img2.ptr;\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, coord_para.yx);\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr;\n\
\n\
    float bval = bias_ptr[gidz];\n\
    float sval = scal_ptr[gidz];\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += vari_ptr[i];\n\
    }\n\
\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = sval * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    bias_val = (bval - scale_vari * mean_vari.s0);\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part0_2x8);\n\
    VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                    uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, tmpData0, src1, 16);\n\
    _viv_asm(COPY, tmpData1, src2, 16);\n\
\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(COPY, src0, norm, 16);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(COPY, src1, norm, 16);\n\
    VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of instance_normalization_scale_f32_bf16_vx*/

static const char instance_normalization_scale_f32_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertEndInt16Fp32_4x4;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_F16F32toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    vxc_short8 src0;\n\
    vxc_half8  in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    Image img3 = create_image_from_image2d(meanVari, 4);\n\
    __global float* bias_ptr = (__global float*)img1.ptr;\n\
    __global float* scal_ptr = (__global float*)img2.ptr;\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, (int2)(0, gidz));\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr;\n\
\n\
    float bval = bias_ptr[gidz];\n\
    float sval = scal_ptr[gidz];\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += vari_ptr[i];\n\
    }\n\
\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = sval * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bval - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
\n\
    coord_in.y ++;\n\
\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndInt16Fp32_4x4);\n\
\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                    VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_F16F32toF16_2D(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(gidz, 0);\n\
    int endH = gidy + height;\n\
    vxc_short8 src0;\n\
    vxc_half8  in_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    Image img3 = create_image_from_image2d(meanVari, 4);\n\
    __global float* bias_ptr = (__global float*)img1.ptr;\n\
    __global float* scal_ptr = (__global float*)img2.ptr;\n\
    __global uchar* sumVari_ptr = (__global uchar*)get_image_ptr_from_coord(img3, coord_para.yx);\n\
    __global float4* vari_ptr = (__global float4*)sumVari_ptr;\n\
\n\
    float bval = bias_ptr[gidz];\n\
    float sval = scal_ptr[gidz];\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += vari_ptr[i];\n\
    }\n\
\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = sval * mean_vari.s1;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    bias_val = (bval - scale_vari * mean_vari.s0);\n\
    vxc_half8 dst;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
\n\
    VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        UniFP16toFP32Lo4_dp4x4);\n\
    VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndInt16Fp32_4x4);\n\
    vxc_float4 norm;\n\
    norm = scale_vari * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = scale_vari * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of instance_normalization_scale_f32_f16_vx*/

static const char instance_normalization_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform float scale_inOut;\n\
_viv_uniform float outputScale;\n\
_viv_uniform int output_ZP;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_U8(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz);\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1 = 0, tmpSqr1 = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord, 0, \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1);\n\
        }\n\
        sqr += (tmpSqr * e2InScale + rowSumScale);\n\
        sum = (tmpSum + sumInZp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_meanvari_U8_2D(\n\
    image2d_array_t input, image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
    int endH = gidy + height;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, 0,\n\
                             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1);\n\
        }\n\
        sqr += (tmpSqr * e2InScale + rowSumScale);\n\
        sum = (tmpSum + sumInZp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_U8toU8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    vxc_uchar16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0,\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord_in, 0, \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.y ++;\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_U8toU8_2D(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    int endH = gidy + height;\n\
    vxc_uchar16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, 0,\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0) * outputScale + output_ZP;\n\
\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
    VXC_ReadImage(src0, input, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = tmpData0 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData1 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    norm = tmpData2 * alpha + bias_val;\n\
    tmpVal0 = convert_int4_rte(norm);\n\
    norm = tmpData3 * alpha + bias_val;\n\
    tmpVal1 = convert_int4_rte(norm);\n\
    VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of instance_normalization_u8_vx*/

static const char instance_normalization_u8_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_U8toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    vxc_uchar16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
\n\
    coord_para = coord;\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(1) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(1) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_para.z, baseAddr);\n\
    for(coord.y = 0; coord.y < height;)\n\
    {\n\
    VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_para.xy = coord.xy;\n\
    coord.y++;\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_para, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    coord_para.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_para, outval, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void instance_norm_U8toF16_2D(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps, int rsFlg)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int4 coord = (int4)(get_global_id(0), gidy, 0, 0);\n\
    int4 coord_para = (int4)(gidz, 0, 0, 0);\n\
    int endH = gidy + height;\n\
    vxc_uchar16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    float scale_vari, bias_val;\n\
    vxc_float4 bias_f, scale_f, mean_vari = (vxc_float4)(0);\n\
\n\
    VXC_ReadImage(src1, scale, coord_para.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, scale_h, src1, 16);\n\
    VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
    bias_f = read_imagef(bias, coord_para);\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_para.yx);\n\
        coord_para.y += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = scale_f.s0 * mean_vari.s1;\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (bias_f.s0 - scale_vari * mean_vari.s0);\n\
    for(; coord.y < endH;)\n\
    {\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord_para = coord;\n\
    coord.y++;\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert2ndUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert3rdUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert4thUint8SubZpToFp32_4x4);\n\
    norm = alpha * tmpData0 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData1 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord_para.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord_para.x += 8;\n\
    norm = alpha * tmpData2 + bias_val;\n\
    _viv_asm(CONV, tmpVal0, norm);\n\
    norm = alpha * tmpData3 + bias_val;\n\
    _viv_asm(CONV, tmpVal1, norm);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, outval, dst, 16);\n\
    VXC_WriteImage(output, coord_para.xy, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of instance_normalization_u8_f16_vx*/

static const char l2normalizescale_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define VXC_Vstore3(Pointer, Offset, Data)   \\\n\
do \\\n\
{ int byteOffset = ((int)sizeof((Data)))*(Offset); \\\n\
VXC_OP3_NoDest(vstore3, Pointer, byteOffset, Data); } \\\n\
while(0)\n\
\n\
#define L2NORMSCALE_SWITCH_PROCESS(case_value, vec_val, ZpValue) \\\n\
                switch (case_value) \\\n\
                { \\\n\
                    case 1: \\\n\
                        vec_val.s123  = ZpValue; \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 2: \\\n\
                        vec_val.s23  = ZpValue; \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 3: \\\n\
                        vec_val.s3  = ZpValue; \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 4: \\\n\
                        vec_val.s4567 = ZpValue; \\\n\
                    break; \\\n\
                    case 5: \\\n\
                        vec_val.s567 = ZpValue; \\\n\
                    break; \\\n\
                    case 6: \\\n\
                        vec_val.s67 = ZpValue; \\\n\
                    break; \\\n\
                    case 7: \\\n\
                        vec_val.s7 = ZpValue; \\\n\
                    break; \\\n\
                    default: \\\n\
                    break; \\\n\
                }\n\
\n\
#define L2NORMSCALE_REM_PROCESS(ZpValue) \\\n\
            VXC_Vload8(src0, src_ptr, 0); \\\n\
            VXC_Vload8(src1, src_ptr, 1); \\\n\
            if (inputRemain <= 8) \\\n\
            { \\\n\
                L2NORMSCALE_SWITCH_PROCESS(inputRemain, src0, ZpValue) \\\n\
                src1 = 0; \\\n\
            } \\\n\
            else if (inputRemain < 16) \\\n\
            { \\\n\
                int inputRemain8 = inputRemain - 8; \\\n\
                L2NORMSCALE_SWITCH_PROCESS(inputRemain8, src1, ZpValue) \\\n\
            }\n\
\n\
\n\
#define L2NORMSCALE_MUL_PROCESS(index) \\\n\
        VXC_Vload8(src0, src_ptr, index); \\\n\
        _viv_asm(COPY, val0, src0, 16); \\\n\
        VXC_Vload8(scale_s16, scale_ptr, index); \\\n\
        _viv_asm(COPY, scale_f16, scale_s16, 16); \\\n\
        _viv_asm(COPY, input_ZP, inputZP, 4); \\\n\
        VXC_DP4x4(vec0, val0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part0_4x4);\\\n\
        VXC_DP4x4(vec1, val0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part1_4x4);\\\n\
        VXC_DP4x4(scale_f32, scale_f16, scale_f16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniFp16toFp32_4x4);\\\n\
        VXC_DP4x4(scale1_f32, scale_f16, scale_f16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniFp16toFp32Hi_4x4);\\\n\
        vec0 = vec0 * rsqrt0.xxxx + output_ZP;\\\n\
        vec1 = vec1 * rsqrt0.xxxx + output_ZP;\\\n\
        vec0 *= scale_f32;\\\n\
        vec1 *= scale1_f32;\\\n\
        _viv_asm(CONV_RTE, dst0, vec0);\\\n\
        _viv_asm(CONV_RTE, dst1, vec1);\\\n\
        VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
        _viv_asm(COPY, dst, dst2, 16);\n\
\n\
_viv_uniform int inputWidth;\n\
_viv_uniform int inputWidthRemain256;\n\
_viv_uniform int inputWidthCount;\n\
_viv_uniform VXC_512Bits uniSumSqrt_16x1;\n\
_viv_uniform float r_inputScale;\n\
\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bin_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform VXC_512Bits uniFp16toFp32Hi_4x4;\n\
_viv_uniform float IntergerScale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform int inputWidthRemain128;\n\
_viv_uniform float zP2x;\n\
_viv_uniform float zpSqrt16x;\n\
_viv_uniform VXC_512Bits uniSumAll_16x1;\n\
_viv_uniform int inputZP;\n\
\n\
#define L2NORMSCALE_MUL_AXIS0_PROCESS(dst_type, convert_type, output_type, copy_type) \\\n\
    vxc_float4 rsqrt0;\\\n\
    Image dst_img = create_image_from_image2d(output, 1); \\\n\
    dst_type  *dst_ptr = (dst_type *)dst_img.ptr; \\\n\
    Image s_img = create_image_from_image2d(scale, 2); \\\n\
    short *scale_ptr = (short *)s_img.ptr; \\\n\
    vxc_float4 vec0, vec1;\\\n\
    convert_type dst0, dst1;\\\n\
    vxc_short8 scale_s16;\\\n\
    vxc_half8  scale_f16;\\\n\
    vxc_float4 scale_f32, scale1_f32;\\\n\
    output_type dst2;\\\n\
    copy_type dst;\\\n\
    rsqrt0 = sum.xxxx * IntergerScale;\\\n\
    src_ptr = src_ptr_base + (get_global_id(0) + get_global_id(1) * inputWidth); \\\n\
    dst_ptr   += (get_global_id(0) + get_global_id(1) * inputWidth);\\\n\
    scale_ptr += get_global_id(0);\\\n\
    for(int i = 0; i < inputWidthCount; i++)\\\n\
    {\\\n\
        L2NORMSCALE_MUL_PROCESS(0) \\\n\
        VXC_Vstore8(dst_ptr, 0, dst); \\\n\
        L2NORMSCALE_MUL_PROCESS(1) \\\n\
        VXC_Vstore8(dst_ptr, 1, dst); \\\n\
        src_ptr   += 256; \\\n\
        dst_ptr   += 256; \\\n\
        scale_ptr += 256; \\\n\
    }\\\n\
    if (inputWidthRemain256) \\\n\
    { \\\n\
        offset  = get_global_id(0) + inputWidthCount * 128; \\\n\
        inputRemain = inputWidth - offset; \\\n\
        if (inputRemain >= 8) \\\n\
        { \\\n\
            L2NORMSCALE_MUL_PROCESS(0) \\\n\
            VXC_Vstore8(dst_ptr, 0, dst); \\\n\
            src_ptr   += 8; \\\n\
            dst_ptr   += 8; \\\n\
            scale_ptr += 8; \\\n\
            inputRemain -= 8; \\\n\
        } \\\n\
        if (inputRemain > 0) \\\n\
        { \\\n\
            L2NORMSCALE_MUL_PROCESS(0) \\\n\
            switch (inputRemain) \\\n\
            { \\\n\
                case 1: \\\n\
                    dst_ptr[0] = dst.s0; \\\n\
                break; \\\n\
                case 2: \\\n\
                    VXC_Vstore2(dst_ptr, 0, dst); \\\n\
                break; \\\n\
                case 3: \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst); \\\n\
                break; \\\n\
                case 4: \\\n\
                    VXC_Vstore4(dst_ptr, 0, dst); \\\n\
                break; \\\n\
                case 5: \\\n\
                    VXC_Vstore2(dst_ptr, 0, dst); \\\n\
                    dst.s012 = dst.s234; \\\n\
                    dst_ptr += 2; \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst); \\\n\
                break; \\\n\
                case 6: \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst); \\\n\
                    dst.s012 = dst.s345; \\\n\
                    dst_ptr += 3; \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst); \\\n\
                break; \\\n\
                case 7: \\\n\
                    VXC_Vstore4(dst_ptr, 0, dst); \\\n\
                     dst.s012 = dst.s456; \\\n\
                    dst_ptr += 4; \\\n\
                    VXC_Vstore3(dst_ptr, 0, dst); \\\n\
                break; \\\n\
                default: \\\n\
                    VXC_Vstore8(dst_ptr, 0, dst); \\\n\
                break; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
\n\
\n\
#define L2NORMSCALE_AXIS0_2D(in0_name, in1_name, out_name, read_type, read_type2, src_type, INPUTSCALE, \\\n\
                            dst_type, convert_type, output_type, copy_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) \\\n\
     void l2normalizescale_axis0_##in0_name##_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_t input,\\\n\
    __read_only  image2d_t scale,\\\n\
    __write_only image2d_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int lidx = get_local_id(0); \\\n\
    int offset  = get_global_id(0); \\\n\
    Image src_img = create_image_from_image2d(input, 1); \\\n\
    read_type *src_ptr_base = (read_type *)src_img.ptr; \\\n\
    read_type *src_ptr; \\\n\
    read_type2 src0, src1; \\\n\
    src_type   val0, val1; \\\n\
    int   inputRemain; \\\n\
    vxc_float4 sum = {0.0f}; \\\n\
    read_type2 input_ZP ;\\\n\
    __local float lcl_sum[16]; \\\n\
    src_ptr = src_ptr_base + (get_global_id(0) + get_global_id(1) * inputWidth); \\\n\
    for (int i = 0; i < inputWidthCount; i++) \\\n\
    { \\\n\
        VXC_Vload8(src0, src_ptr, 0); \\\n\
        VXC_Vload8(src1, src_ptr, 1); \\\n\
        _viv_asm(COPY, val0, src0, 16); \\\n\
        _viv_asm(COPY, val1, src1, 16); \\\n\
        VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
            uniSumSqrt_16x1); \\\n\
        sum.x += sum.y;  \\\n\
        src_ptr += 256; \\\n\
    } \\\n\
    if (inputWidthRemain256) \\\n\
    { \\\n\
        offset  = get_global_id(0) + inputWidthCount * 256;\\\n\
        inputRemain = inputWidth - offset; \\\n\
        if (inputRemain > 0) \\\n\
        { \\\n\
            L2NORMSCALE_REM_PROCESS(0) \\\n\
            _viv_asm(COPY, val0, src0, 16); \\\n\
            _viv_asm(COPY, val1, src1, 16); \\\n\
            VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
                uniSumSqrt_16x1); \\\n\
            sum.x += sum.y; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum.x; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    sum.x = dot(data0, one); \\\n\
    sum.x = rsqrt(sum.x) * INPUTSCALE; \\\n\
    L2NORMSCALE_MUL_AXIS0_PROCESS(dst_type, convert_type, output_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS0_2D(F16, F16, F16, ushort, vxc_ushort8, vxc_half8, 1, \\\n\
                     ushort, half4, vxc_half8, vxc_ushort8)\n\
L2NORMSCALE_AXIS0_2D(I16, F16, F16, short,  vxc_short8, vxc_short8, r_inputScale, \\\n\
                     ushort, half4, vxc_half8, vxc_ushort8)\n\
L2NORMSCALE_AXIS0_2D(I16, F16, I16, short,  vxc_short8, vxc_short8, r_inputScale, \\\n\
                     short, int4, vxc_short8, vxc_short8)\n\
L2NORMSCALE_AXIS0_2D(I8,  F16, F16, char,   vxc_char8,  vxc_char8, r_inputScale, \\\n\
                     ushort, half4, vxc_half8, vxc_ushort8)\n\
L2NORMSCALE_AXIS0_2D(I8,  F16, I8,  char,   vxc_char8,  vxc_char8, r_inputScale, \\\n\
                     char,  int4, vxc_char8, vxc_char8)\n\
\n\
\n\
\n\
#define L2NORMSCALE_AXIS0_U8_2D(in1_name, out_name,\\\n\
                            dst_type, convert_type, output_type, copy_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) \\\n\
     void l2normalizescale_axis0_U8_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_t input,\\\n\
    __read_only  image2d_t scale,\\\n\
    __write_only image2d_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int lidx = get_local_id(0); \\\n\
    int offset  = get_global_id(0); \\\n\
    Image src_img = create_image_from_image2d(input, 1);\n\
    uchar *src_ptr_base = (uchar *)src_img.ptr; \\\n\
    uchar *src_ptr; \\\n\
    vxc_uchar8 src0, src1; \\\n\
    vxc_uchar8   val0, val1; \\\n\
    int   inputRemain; \\\n\
    vxc_float4 sum = {0.0f}; \\\n\
    vxc_uchar8 input_ZP ; \\\n\
    __local float lcl_sum[16]; \\\n\
    src_ptr = src_ptr_base + (get_global_id(0) + get_global_id(1) * inputWidth); \\\n\
    for (int i = 0; i < inputWidthCount; i++) \\\n\
    { \\\n\
        VXC_Vload8(src0, src_ptr, 0); \\\n\
        VXC_Vload8(src1, src_ptr, 1); \\\n\
        _viv_asm(COPY, val0, src0, 16); \\\n\
        _viv_asm(COPY, val1, src1, 16); \\\n\
        VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
            uniSumSqrt_16x1); \\\n\
        VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 1),\\\n\
            uniSumAll_16x1); \\\n\
        sum.w  = sum.y - zP2x * sum.z + zpSqrt16x; \\\n\
        sum.x += sum.w; \\\n\
        src_ptr += 256; \\\n\
    } \\\n\
    if (inputWidthRemain256) \\\n\
    { \\\n\
        offset  = get_global_id(0) + inputWidthCount * 256; \\\n\
        inputRemain = inputWidth - offset; \\\n\
        if (inputRemain > 0) \\\n\
        { \\\n\
            L2NORMSCALE_REM_PROCESS((uchar)inputZP) \\\n\
            _viv_asm(COPY, val0, src0, 16); \\\n\
            _viv_asm(COPY, val1, src1, 16); \\\n\
            VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 1),\\\n\
                uniSumSqrt_16x1); \\\n\
            VXC_DP16x1(sum, val0, val1, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 1),\\\n\
                uniSumAll_16x1); \\\n\
            sum.w  = sum.y - zP2x * sum.z + zpSqrt16x; \\\n\
            sum.x += sum.w; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum.x; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0]; \\\n\
    float4 one = (float4)(1, 1, 1, 1); \\\n\
    float4 data0; \\\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3]; \\\n\
    sum.x = dot(data0, one); \\\n\
    sum.x = rsqrt(sum.x) * r_inputScale; \\\n\
    L2NORMSCALE_MUL_AXIS0_PROCESS(dst_type, convert_type, output_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS0_U8_2D(F16, F16, ushort, half4, vxc_half8,  vxc_ushort8)\n\
L2NORMSCALE_AXIS0_U8_2D(F16, U8,  uchar,  int4,  vxc_uchar8, vxc_uchar8)\n\
"; /* end of l2normalizescale_axis0_vx*/

static const char l2normalizescale_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************************************L2NormalizeScale*****************************************/\n\
_viv_uniform int L2NorS_depth;\n\
_viv_uniform VXC_512Bits UniFp16MulLo_dp4x4;\n\
_viv_uniform VXC_512Bits UniFp16MulHi_dp4x4;\n\
\n\
//int8 version\n\
_viv_uniform float r_inputScale;\n\
_viv_uniform VXC_512Bits uniIntegerSquareLo_4x4;\n\
_viv_uniform VXC_512Bits uniIntegerSquareHi_4x4;\n\
_viv_uniform VXC_512Bits uniDataSquareAddU32Lo_4x4;\n\
_viv_uniform VXC_512Bits uniDataSquareAddU32Hi_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniUInt8SquareLo_4x4;\n\
_viv_uniform VXC_512Bits uniUInt8SquareHi_4x4;\n\
_viv_uniform int inputZP;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bin_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float IntergerScale;\n\
_viv_uniform float output_ZP;\n\
\n\
#define L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
    coord.y = get_global_id(1); \\\n\
    input_type  vect0, vect1;\\\n\
    incopy_type src0, src1;\\\n\
    vxc_float4 rsqrt0, rsqrt1;\\\n\
    rsqrt0 = sum_lo;\\\n\
    rsqrt1 = sum_hi;\\\n\
    rsqrt0 *= IntergerScale;\\\n\
    rsqrt1 *= IntergerScale;\\\n\
    for(int i = 0; i < L2NorS_depth; i += 2)\\\n\
   {\\\n\
        vxc_float4 vec0, vec1;\\\n\
        input_type input_ZP ;\\\n\
        convert_type dst0, dst1;\\\n\
        VXC_ReadImage(vect0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        _viv_asm(COPY, src0, vect0, 16); \\\n\
        VXC_ReadImage(vect1, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        _viv_asm(COPY, src1, vect1, 16); \\\n\
        vxc_short8 scale_s16;\\\n\
        vxc_half8  scale_f16;\\\n\
        vxc_float4 scale_f32;\\\n\
        VXC_ReadImage(scale_s16, scale, coord.yw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\\\n\
        _viv_asm(COPY, scale_f16, scale_s16, 16); \\\n\
        _viv_asm(COPY, input_ZP, inputZP, 4); \\\n\
        VXC_DP4x4(vec0, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part0_4x4);\\\n\
        VXC_DP4x4(vec1, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part1_4x4);\\\n\
        VXC_DP4x4(scale_f32, scale_f16, scale_f16, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardInf, 0),\\\n\
            uniFp16toFp32_4x4);\\\n\
        vec0 = vec0 * rsqrt0;\\\n\
        vec1 = vec1 * rsqrt1;\\\n\
        vec0 = vec0 * scale_f32.xxxx + output_ZP;\\\n\
        vec1 = vec1 * scale_f32.xxxx + output_ZP;\\\n\
        _viv_asm(CONV_RTE, dst0, vec0);\\\n\
        _viv_asm(CONV_RTE, dst1, vec1);\\\n\
        output_type dst2;\\\n\
        VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
        copy_type dst;\\\n\
        _viv_asm(COPY, dst, dst2, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        VXC_DP4x4(vec0, src1, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part0_4x4);\\\n\
        VXC_DP4x4(vec1, src1, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0),\\\n\
            uniDataSubZPtoFp32Part1_4x4);\\\n\
        vec0 = vec0 * rsqrt0;\\\n\
        vec1 = vec1 * rsqrt1;\\\n\
        vec0 = vec0 * scale_f32.yyyy + output_ZP;\\\n\
        vec1 = vec1 * scale_f32.yyyy + output_ZP;\\\n\
        _viv_asm(CONV_RTE, dst0, vec0);\\\n\
        _viv_asm(CONV_RTE, dst1, vec1);\\\n\
        VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
        coord.y++;\\\n\
        _viv_asm(COPY, dst, dst2, 16); \\\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
        coord.y++;\\\n\
    }\\\n\
\n\
\n\
#define L2NORMSCALE_AXIS1_F16_2D(in1_name, out_name,\\\n\
       input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
__kernel void l2normalizescale_axis1_F16_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_array_t input,\\\n\
    __read_only  image2d_array_t scale,\\\n\
    __write_only image2d_array_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 img1_s16, img2_s16; \\\n\
    vxc_float4 squr, sum_lo = 0, sum_hi = 0; \\\n\
    vxc_half8 img1_fp16, img2_fp16; \\\n\
    for(int i = 0; i < L2NorS_depth; i += 2) \\\n\
    { \\\n\
        VXC_ReadImage(img1_s16, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(img2_s16, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y += 2; \\\n\
        _viv_asm(COPY, img1_fp16, img1_s16, 16); \\\n\
        _viv_asm(COPY, img2_fp16, img2_s16, 16); \\\n\
        VXC_DP4x4(squr, img1_fp16, img1_fp16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulLo_dp4x4); \\\n\
        sum_lo += squr; \\\n\
        VXC_DP4x4(squr, img2_fp16, img2_fp16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulLo_dp4x4); \\\n\
        sum_lo += squr; \\\n\
        VXC_DP4x4(squr, img1_fp16, img1_fp16, VXC_MODIFIER(0, 3, 4, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulHi_dp4x4); \\\n\
        sum_hi += squr; \\\n\
        VXC_DP4x4(squr, img2_fp16, img2_fp16, VXC_MODIFIER(0, 3, 4, VXC_RM_TowardZero, 1),\\\n\
            UniFp16MulHi_dp4x4); \\\n\
        sum_hi += squr; \\\n\
    } \\\n\
    sum_lo = rsqrt(sum_lo); \\\n\
    sum_hi = rsqrt(sum_hi); \\\n\
    L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS1_F16_2D(F16, F16, vxc_short8,  vxc_half8,   vxc_half8,   half4,        vxc_short8)\n\
\n\
\n\
#define L2NORMSCALE_AXIS1_I8_2D(in1_name, out_name,\\\n\
       input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
__kernel void l2normalizescale_axis1_I8_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_array_t input,\\\n\
    __read_only  image2d_array_t scale,\\\n\
    __write_only image2d_array_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_char8 src0_I8, src1_I8; \\\n\
    vxc_uint4 dst0_I8 = 0, dst1_I8 = 0; \\\n\
    for(int i = 0; i < L2NorS_depth; i += 2) \\\n\
    { \\\n\
        VXC_ReadImage(src0_I8, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1_I8, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y += 2; \\\n\
        VXC_DP4x4(dst0_I8, src0_I8, dst0_I8, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataSquareAddU32Lo_4x4); \\\n\
        VXC_DP4x4(dst1_I8, src0_I8, dst1_I8, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataSquareAddU32Hi_4x4); \\\n\
        VXC_DP4x4(dst0_I8, src1_I8, dst0_I8, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataSquareAddU32Lo_4x4); \\\n\
        VXC_DP4x4(dst1_I8, src1_I8, dst1_I8, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniDataSquareAddU32Hi_4x4); \\\n\
    } \\\n\
    vxc_float4 sum_lo, sum_hi; \\\n\
    sum_lo = convert_float4(dst0_I8); \\\n\
    sum_hi = convert_float4(dst1_I8); \\\n\
    sum_lo = rsqrt(sum_lo) * r_inputScale; \\\n\
    sum_hi = rsqrt(sum_hi) * r_inputScale; \\\n\
    L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS1_I8_2D(F16, I8,  vxc_char16,  vxc_char16,  vxc_char16,  int4,   vxc_char16)\n\
L2NORMSCALE_AXIS1_I8_2D(F16, F16, vxc_char16,  vxc_char16,  vxc_half8,   half4,  vxc_short8)\n\
\n\
\n\
#define L2NORMSCALE_AXIS1_I16_2D(in1_name, out_name,\\\n\
       input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
__kernel void l2normalizescale_axis1_I16_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_array_t input,\\\n\
    __read_only  image2d_array_t scale,\\\n\
    __write_only image2d_array_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 src0_I16, src1_I16; \\\n\
    vxc_float4 squr, sum_lo = 0, sum_hi = 0; \\\n\
    for(int i = 0; i < L2NorS_depth; i += 2) \\\n\
    { \\\n\
        VXC_ReadImage(src0_I16, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1_I16, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y += 2; \\\n\
        VXC_DP4x4(squr, src0_I16, src0_I16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniIntegerSquareLo_4x4); \\\n\
        sum_lo = squr + sum_lo; \\\n\
        VXC_DP4x4(squr, src0_I16, src0_I16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniIntegerSquareHi_4x4); \\\n\
        sum_hi = squr + sum_hi; \\\n\
        VXC_DP4x4(squr, src1_I16, src1_I16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniIntegerSquareLo_4x4); \\\n\
        sum_lo = squr + sum_lo; \\\n\
        VXC_DP4x4(squr, src1_I16, src1_I16, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniIntegerSquareHi_4x4); \\\n\
        sum_hi = squr + sum_hi; \\\n\
    } \\\n\
    sum_lo = rsqrt(sum_lo) * r_inputScale; \\\n\
    sum_hi = rsqrt(sum_hi) * r_inputScale; \\\n\
    L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS1_I16_2D(F16, I16, vxc_short8, vxc_short8, vxc_short8, int4,  vxc_short8)\n\
L2NORMSCALE_AXIS1_I16_2D(F16, F16, vxc_short8, vxc_short8, vxc_half8,  half4, vxc_short8)\n\
\n\
#define L2NORMSCALE_AXIS1_U8_2D(in1_name, out_name,\\\n\
       input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
__kernel void l2normalizescale_axis1_U8_##in1_name##to##out_name##_2D \\\n\
    (\\\n\
    __read_only  image2d_array_t input,\\\n\
    __read_only  image2d_array_t scale,\\\n\
    __write_only image2d_array_t output,\\\n\
    int axis\\\n\
    )\\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar8 src0_U8, src1_U8; \\\n\
    vxc_float4 squr, sum_lo = 0, sum_hi = 0; \\\n\
    for(int i = 0; i < L2NorS_depth; i += 2) \\\n\
    { \\\n\
        vxc_uchar8 zero; \\\n\
        VXC_ReadImage(src0_U8, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_ReadImage(src1_U8, input, coord.xy, VXC_5BITOFFSET_XY(0, 1),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y += 2; \\\n\
        _viv_asm(COPY, zero, inputZP, 4); \\\n\
        VXC_DP4x4(squr, src0_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareLo_4x4); \\\n\
        sum_lo = squr + sum_lo; \\\n\
        VXC_DP4x4(squr, src0_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareHi_4x4); \\\n\
        sum_hi = squr + sum_hi; \\\n\
        VXC_DP4x4(squr, src1_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareLo_4x4); \\\n\
        sum_lo = squr + sum_lo; \\\n\
        VXC_DP4x4(squr, src1_U8, zero, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniUInt8SquareHi_4x4); \\\n\
        sum_hi = squr + sum_hi; \\\n\
    } \\\n\
    sum_lo = rsqrt(sum_lo) * r_inputScale; \\\n\
    sum_hi = rsqrt(sum_hi) * r_inputScale; \\\n\
    L2NORMSCALE_MUL_AXIS1_PROCESS(input_type, incopy_type, output_type, convert_type, copy_type) \\\n\
}\n\
\n\
L2NORMSCALE_AXIS1_U8_2D(F16, F16, vxc_uchar16, vxc_uchar16, vxc_half8,   half4,  vxc_short8)\n\
L2NORMSCALE_AXIS1_U8_2D(F16, U8,  vxc_uchar16, vxc_uchar16, vxc_uchar16, int4,   vxc_uchar16)\n\
"; /* end of l2normalizescale_axis1_vx*/

static const char layer_normalization_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************layernorm float16***********************************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_dp4x4;\n\
\n\
__kernel void layer_norm_F16toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        vxc_half8  val0_h;\n\
        _viv_asm(COPY, val0_h, src0, 16);\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, val0_h, val0_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
            uniFp16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr += sumsqr.y;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f;\n\
    for(coord.x = 0; coord.x < width; coord.x += 4)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        bias_f = read_imagef(bias, coord.xw);\n\
        vxc_half8 in_h, scale_h;\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        vxc_float4 in_f, scale_f;\n\
        VXC_DP4x4(in_f, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        vxc_float4 sub, norm;\n\
        sub = in_f - mean;\n\
        norm = scale_f * vari * sub + bias_f;\n\
        half4 norm_h;\n\
        _viv_asm(CONV, norm_h, norm);\n\
        vxc_half8 dst;\n\
        VXC_DP4x4(dst, norm_h, norm_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniExtractHalf4_dp4x4);\n\
        vxc_short8 dstval;\n\
        _viv_asm(COPY, dstval, dst, 16);\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dstval, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
/*****************************layernorm uint8 to uint8****************************/\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform int tmpZp2;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
__kernel void layer_norm_U8toU8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
\n\
    vxc_uchar16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    vxc_int4 tmpSum1;\n\
    vxc_int4 tmpSqr1;\n\
    short zp = inputZP;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
        tmpSum += (tmpSum1.x);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
        tmpSqr += (tmpSqr1.x + tmpZp1 * tmpSum1.x);\n\
    }\n\
    sum = (tmpSum + sumInZp) * input_scale;\n\
    sqr = (tmpSqr + tmpZp2) * e2InScale;\n\
\n\
    float mean, vari;\n\
    mean = sum * dimRatio;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3;\n\
    int2 coord_bias = (int2)(0, 0);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.x = coord.x;\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(8, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert3rdUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert4thUint8SubZpToFp32_4x4);\n\
        tmpData0 *= input_scale;\n\
        tmpData1 *= input_scale;\n\
        tmpData2 *= input_scale;\n\
        tmpData3 *= input_scale;\n\
\n\
        vxc_float4 norm;\n\
        tmpData0 -= mean;\n\
        norm = scale_f0 * vari * tmpData0 + bias_f0;\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        coord_bias.x += 4;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        tmpData1 -= mean;\n\
        norm = scale_f1 * vari * tmpData1 + bias_f1;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
\n\
        tmpData2 -= mean;\n\
        norm = scale_f0 * vari * tmpData2 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        tmpData3 -= mean;\n\
        norm = scale_f1 * vari * tmpData3 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, src2, \\\n\
                VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
/***************************layernorm float16 to uint8**************************/\n\
__kernel void layer_norm_F16toU8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        vxc_half8  val0_h;\n\
        _viv_asm(COPY, val0_h, src0, 16);\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, val0_h, val0_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
            uniFp16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr += sumsqr.y;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f;\n\
    for(coord.x = 0; coord.x < width; coord.x += 4)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        bias_f = read_imagef(bias, coord.xw);\n\
        vxc_half8 in_h, scale_h;\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        vxc_float4 in_f, scale_f;\n\
        VXC_DP4x4(in_f, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        vxc_float4 sub, norm;\n\
        sub = in_f - mean;\n\
        norm = scale_f * vari * sub + bias_f;\n\
        norm = norm * outputScale + output_zp;\n\
        int4 output_int4;\n\
        output_int4 = convert_int4_rte(norm);\n\
        vxc_uchar8 dst;\n\
        VXC_DP2x8(dst, output_int4, output_int4, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\n\
            uniConvertInt32toUint8_2x8);\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dst, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_vx*/

static const char layer_normalization_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************layernorm float16***********************************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_dp4x4;\n\
\n\
__kernel void layer_norm_F16toF16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_short8 src0, src1;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        vxc_half8  val0_h;\n\
        _viv_asm(COPY, val0_h, src0, 16);\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, val0_h, val0_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
            uniFp16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr += sumsqr.y;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f;\n\
    for(coord.x = 0; coord.x < width; coord.x += 4)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        bias_f = read_imagef(bias, coord.xw);\n\
        vxc_half8 in_h, scale_h;\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        vxc_float4 in_f, scale_f;\n\
        VXC_DP4x4(in_f, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        vxc_float4 sub, norm;\n\
        sub = in_f - mean;\n\
        norm = scale_f * vari * sub + bias_f;\n\
        half4 norm_h;\n\
        _viv_asm(CONV, norm_h, norm);\n\
        vxc_half8 dst;\n\
        VXC_DP4x4(dst, norm_h, norm_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniExtractHalf4_dp4x4);\n\
        vxc_short8 dstval;\n\
        _viv_asm(COPY, dstval, dst, 16);\n\
        VXC_WriteImage(output, coord.xy, dstval, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
/*****************************layernorm uint8 to uint8****************************/\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform int tmpZp2;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
__kernel void layer_norm_U8toU8_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_uchar16 src0, src2;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    vxc_int4 tmpSum1;\n\
    vxc_int4 tmpSqr1;\n\
    short zp = inputZP;\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
        tmpSum += (tmpSum1.x);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
        tmpSqr += (tmpSqr1.x + tmpZp1 * tmpSum1.x);\n\
    }\n\
    sum = (tmpSum + sumInZp) * input_scale;\n\
    sqr = (tmpSqr + tmpZp2) * e2InScale;\n\
\n\
    float mean, vari;\n\
    mean = sum * dimRatio;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3;\n\
    int2 coord_bias = (int2)(0, 0);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.x = coord.x;\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(8, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert3rdUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert4thUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean;\n\
        tmpData1 = tmpData1 * input_scale - mean;\n\
        tmpData2 = tmpData2 * input_scale - mean;\n\
        tmpData3 = tmpData3 * input_scale - mean;\n\
\n\
        vxc_float4 norm;\n\
        norm = scale_f0 * vari * tmpData0 + bias_f0;\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        coord_bias.x += 4;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        norm = scale_f1 * vari * tmpData1 + bias_f1;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
\n\
        norm = scale_f0 * vari * tmpData2 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        norm = scale_f1 * vari * tmpData3 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
/***************************layernorm float16 to uint8**************************/\n\
__kernel void layer_norm_F16toU8_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_short8 src0, src1;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        vxc_half8  val0_h;\n\
        _viv_asm(COPY, val0_h, src0, 16);\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, val0_h, val0_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
            uniFp16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr += sumsqr.y;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f;\n\
    for(coord.x = 0; coord.x < width; coord.x += 4)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_ReadImage(src1, scale, coord.xw, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        bias_f = read_imagef(bias, coord.xw);\n\
        vxc_half8 in_h, scale_h;\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        vxc_float4 in_f, scale_f;\n\
        VXC_DP4x4(in_f, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        vxc_float4 sub, norm;\n\
        sub = in_f - mean;\n\
        norm = scale_f * vari * sub + bias_f;\n\
        norm = norm * outputScale + output_zp;\n\
        int4 output_int4;\n\
        output_int4 = convert_int4_rte(norm);\n\
        vxc_uchar8 dst;\n\
        VXC_DP2x8(dst, output_int4, output_int4, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\n\
            uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of layer_normalization_2d_vx*/

static const char layer_normalization_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************layernorm float16***********************************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform float dimRatio_scale;\n\
_viv_uniform VXC_512Bits uniInt16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
\n\
__kernel void layer_norm_I16toI16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_in = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_float sum = 0, sqr = 0;\n\
    for(; coord_in.x < width;)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x += 8;\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniInt16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr = sqr + sumsqr.y * e2InScale;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio_scale;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_half8 scale_h;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    int2 coord_bias = (int2)(0, 0);\n\
\n\
    for(coord_in.x = 0; coord_in.x < width; coord_in.x += 8, coord.x += 8)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.x = coord_in.x;\n\
        VXC_ReadImage(src1, scale, coord_bias, VXC_5BITOFFSET_XY(0, 0),\\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 * input_scale - mean;\n\
        norm = scale_f0 * vari * sub + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        sub = tmpData1 * input_scale - mean;\n\
        norm = scale_f1 * vari * sub + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, dst, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void layer_norm_I16toI16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int2 coord = (int2)(0, get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_float sum = 0, sqr = 0;\n\
    for(; coord.x < width;)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniInt16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr = sqr + sumsqr.y * e2InScale;\n\
    }\n\
    vxc_float mean, vari;\n\
    mean = sum * dimRatio_scale;\n\
    vari = sqr * dimRatio - mean * mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_half8 scale_h;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    int2 coord_bias = (int2)(0, 0);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.x = coord.x;\n\
        VXC_ReadImage(src1, scale, coord_bias, VXC_5BITOFFSET_XY(0, 0),\\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                    UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniConvertSecFp16Fp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 * input_scale - mean;\n\
        norm = scale_f0 * vari * sub + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        sub = tmpData1 * input_scale - mean;\n\
        norm = scale_f1 * vari * sub + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                    uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of layer_normalization_i16_vx*/

static const char layer_normalization_scale_f32_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************layernorm float16***********************************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_dp4x4;\n\
\n\
__kernel void layer_norm_F16F32toF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    vxc_short8 src0;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        vxc_half8  val0_h;\n\
        _viv_asm(COPY, val0_h, src0, 16);\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, val0_h, val0_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
            uniFp16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr += sumsqr.y;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f, scale_f, in_f;\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, (int2)(0, 0));\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, (int2)(0, 0));\n\
    for(coord.x = 0; coord.x < width; coord.x += 4)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        bias_f = vload4(0, bias_ptr + coord.x);\n\
        scale_f = vload4(0, scale_ptr + coord.x);\n\
        vxc_half8 in_h;\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP4x4(in_f, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        vxc_float4 sub, norm;\n\
        sub = in_f - mean;\n\
        norm = scale_f * vari * sub + bias_f;\n\
        half4 norm_h;\n\
        _viv_asm(CONV, norm_h, norm);\n\
        vxc_half8 dst;\n\
        VXC_DP4x4(dst, norm_h, norm_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniExtractHalf4_dp4x4);\n\
        vxc_short8 dstval;\n\
        _viv_asm(COPY, dstval, dst, 16);\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, dstval, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
/*****************************layernorm uint8 to uint8****************************/\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform int tmpZp2;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniInt16SumSqr_dp8x2;\n\
_viv_uniform float dimRatio_scale;\n\
\n\
__kernel void layer_norm_U8F32toU8(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
\n\
    vxc_uchar16 src0, src2;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 bias_f0, bias_f1, bias_f2, bias_f3, scale_f0, scale_f1, scale_f2, scale_f3;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    vxc_int4 tmpSum1;\n\
    vxc_int4 tmpSqr1;\n\
    short zp = inputZP;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
        tmpSum += (tmpSum1.x);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
        tmpSqr += (tmpSqr1.x + tmpZp1 * tmpSum1.x);\n\
    }\n\
    sum = (tmpSum + sumInZp) * input_scale;\n\
    sqr = (tmpSqr + tmpZp2) * e2InScale;\n\
\n\
    float mean, vari;\n\
    mean = sum * dimRatio;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3;\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, (int2)(0, 0));\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, (int2)(0, 0));\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        bias_f2 = vload4(2, bias_ptr);\n\
        bias_f3 = vload4(3, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        scale_f2 = vload4(2, scale_ptr);\n\
        scale_f3 = vload4(3, scale_ptr);\n\
        bias_ptr += 16;\n\
        scale_ptr += 16;\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert3rdUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert4thUint8SubZpToFp32_4x4);\n\
        tmpData0 *= input_scale;\n\
        tmpData1 *= input_scale;\n\
        tmpData2 *= input_scale;\n\
        tmpData3 *= input_scale;\n\
\n\
        vxc_float4 norm;\n\
        tmpData0 -= mean;\n\
        norm = scale_f0 * vari * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        tmpData1 -= mean;\n\
        norm = scale_f1 * vari * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
\n\
        tmpData2 -= mean;\n\
        norm = scale_f2 * vari * tmpData2 + bias_f2;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        tmpData3 -= mean;\n\
        norm = scale_f3 * vari * tmpData3 + bias_f3;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, src2, \\\n\
                VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void layer_norm_I16F32toI16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_in = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_short8 src0, dst;\n\
    vxc_float sum = 0, sqr = 0;\n\
    for(; coord_in.x < width;)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x += 8;\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniInt16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr = sqr + sumsqr.y * e2InScale;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio_scale;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    int2 coord_bias = (int2)(0, 0);\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, coord_bias);\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, coord_bias);\n\
    for(coord_in.x = 0; coord_in.x < width; coord_in.x += 8, coord.x += 8)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        bias_ptr += 8;\n\
        scale_ptr += 8;\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 * input_scale - mean;\n\
        norm = scale_f0 * vari * sub + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        sub = tmpData1 * input_scale - mean;\n\
        norm = scale_f1 * vari * sub + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, dst, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_scale_f32_vx*/

static const char layer_normalization_scale_f32_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************layernorm float16***********************************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_dp4x4;\n\
\n\
__kernel void layer_norm_F16F32toF16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_short8 src0, src1;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        vxc_half8  val0_h;\n\
        _viv_asm(COPY, val0_h, src0, 16);\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, val0_h, val0_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
            uniFp16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr += sumsqr.y;\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f, scale_f, in_f;\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, coord.zw);\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, coord.zw);\n\
    for(coord.x = 0; coord.x < width; coord.x += 4)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        bias_f = vload4(0, bias_ptr + coord.x);\n\
        scale_f = vload4(0, scale_ptr + coord.x);\n\
\n\
        vxc_half8 in_h;\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP4x4(in_f, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        vxc_float4 sub, norm;\n\
        sub = in_f - mean;\n\
        norm = scale_f * vari * sub + bias_f;\n\
        half4 norm_h;\n\
        _viv_asm(CONV, norm_h, norm);\n\
        vxc_half8 dst;\n\
        VXC_DP4x4(dst, norm_h, norm_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniExtractHalf4_dp4x4);\n\
        vxc_short8 dstval;\n\
        _viv_asm(COPY, dstval, dst, 16);\n\
        VXC_WriteImage(output, coord.xy, dstval, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
/*****************************layernorm uint8 to uint8****************************/\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform int tmpZp2;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniInt16SumSqr_dp8x2;\n\
_viv_uniform float dimRatio_scale;\n\
\n\
__kernel void layer_norm_U8F32toU8_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_uchar16 src0, src2;\n\
    float sum = 0, sqr = 0;\n\
    vxc_float4 bias_f0, bias_f1, bias_f2, bias_f3, scale_f0, scale_f1, scale_f2, scale_f3;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    vxc_int4 tmpSum1;\n\
    vxc_int4 tmpSqr1;\n\
    short zp = inputZP;\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
        tmpSum += (tmpSum1.x);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
        tmpSqr += (tmpSqr1.x + tmpZp1 * tmpSum1.x);\n\
    }\n\
    sum = (tmpSum + sumInZp) * input_scale;\n\
    sqr = (tmpSqr + tmpZp2) * e2InScale;\n\
\n\
    float mean, vari;\n\
    mean = sum * dimRatio;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3;\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, coord.zw);\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, coord.zw);\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        bias_f2 = vload4(2, bias_ptr);\n\
        bias_f3 = vload4(3, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        scale_f2 = vload4(2, scale_ptr);\n\
        scale_f3 = vload4(3, scale_ptr);\n\
        bias_ptr += 16;\n\
        scale_ptr += 16;\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert3rdUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert4thUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean;\n\
        tmpData1 = tmpData1 * input_scale - mean;\n\
        tmpData2 = tmpData2 * input_scale - mean;\n\
        tmpData3 = tmpData3 * input_scale - mean;\n\
\n\
        vxc_float4 norm;\n\
        norm = scale_f0 * vari * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        norm = scale_f1 * vari * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
\n\
        norm = scale_f2 * vari * tmpData2 + bias_f2;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        norm = scale_f3 * vari * tmpData3 + bias_f3;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(src2, tmpVal0, tmpVal1, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void layer_norm_I16F32toI16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_float sum = 0, sqr = 0;\n\
    for(; coord.x < width;)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
        vxc_float4 sumsqr;\n\
        VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniInt16SumSqr_dp8x2);\n\
        sum += sumsqr.x;\n\
        sqr = sqr + sumsqr.y * e2InScale;\n\
    }\n\
    vxc_float mean, vari;\n\
    mean = sum * dimRatio_scale;\n\
    vari = sqr * dimRatio - mean * mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_half8 scale_h;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, coord.zw);\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, coord.zw);\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        bias_ptr += 8;\n\
        scale_ptr += 8;\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 * input_scale - mean;\n\
        norm = scale_f0 * vari * sub + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        sub = tmpData1 * input_scale - mean;\n\
        norm = scale_f1 * vari * sub + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                    uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_scale_f32_2d_vx*/

static const char layer_normalization_scale_f32_bf16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/**************************layernorm float16***********************************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void layer_norm_BF16F32toBF16(\n\
    image2d_array_t input, image2d_t bias, image2d_t scale,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 ones = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    vxc_ushort8 src0, src1, src2;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_OP4(img_load_3d, src0, input, coord.xyzz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
    float4 srcA, srcB;\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        sum += dot(srcA, ones) + dot(srcB, ones);\n\
        sqr += dot(srcA * srcA, ones) + dot(srcB * srcB, ones);\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, (int2)(0, 0));\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, (int2)(0, 0));\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        bias_ptr += 8;\n\
        scale_ptr += 8;\n\
\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
\n\
\n\
        vxc_float4 sub0, sub1, norm0, norm1;\n\
        sub0 = srcA - mean;\n\
        sub1 = srcB - mean;\n\
        norm0 = scale_f0 * vari * sub0 + bias_f0;\n\
        norm1 = scale_f1 * vari * sub1 + bias_f1;\n\
\n\
        _viv_asm(COPY, src0, norm0, 16);\n\
        _viv_asm(COPY, src1, norm1, 16);\n\
        VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, src2, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void layer_norm_BF16F32toBF16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale,\n\
    image2d_t output, float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_ushort8 src0, src1, src2;\n\
    vxc_float sum = 0, sqr = 0;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    float4 ones = (float4)(1.0, 1.0, 1.0, 1.0);\n\
    Image img1 = create_image_from_image2d(bias, 4);\n\
    Image img2 = create_image_from_image2d(scale, 4);\n\
    float4 srcA, srcB;\n\
    for(coord.x = 8; coord.x < (width+8); coord.x += 8)\n\
    {\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        sum += dot(srcA, ones) + dot(srcB, ones);\n\
        sqr += dot(srcA * srcA, ones) + dot(srcB * srcB, ones);\n\
    }\n\
    vxc_float mean;\n\
    mean = sum * dimRatio;\n\
    vxc_float vari;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    __global float* bias_ptr = (__global float*)get_image_ptr_from_coord(img1, coord.zw);\n\
    __global float* scale_ptr = (__global float*)get_image_ptr_from_coord(img2, coord.zw);\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = vload4(0, bias_ptr);\n\
        bias_f1 = vload4(1, bias_ptr);\n\
        scale_f0 = vload4(0, scale_ptr);\n\
        scale_f1 = vload4(1, scale_ptr);\n\
        bias_ptr += 8;\n\
        scale_ptr += 8;\n\
\n\
        VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part0_2x8);\n\
        VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                     uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, srcA, src1, 16);\n\
        _viv_asm(COPY, srcB, src2, 16);\n\
\n\
        vxc_float4 sub0, sub1, norm0, norm1;\n\
        sub0 = srcA - mean;\n\
        sub1 = srcB - mean;\n\
        norm0 = scale_f0 * vari * sub0 + bias_f0;\n\
        norm1 = scale_f1 * vari * sub1 + bias_f1;\n\
\n\
        _viv_asm(COPY, src0, norm0, 16);\n\
        _viv_asm(COPY, src1, norm1, 16);\n\
        VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord.xy, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_scale_f32_bf16_vx*/

static const char layer_normalization_u8_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/*****************************layernorm uint8 to fp16****************************/\n\
_viv_uniform int width;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert3rdUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert4thUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform int tmpZp2;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform VXC_512Bits UniPackFP16even_2x8;\n\
\n\
__kernel void layer_norm_U8toF16(\n\
    image2d_array_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_array_t output,\n\
              float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), get_global_id(2));\n\
    int4 coord_out = coord;\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    vxc_int4 tmpSum1;\n\
    vxc_int4 tmpSqr1;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.z, baseAddr);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
        tmpSum += (tmpSum1.x);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
        tmpSqr += (tmpSqr1.x + tmpZp1 * tmpSum1.x);\n\
    }\n\
    sum = (tmpSum + sumInZp) * input_scale;\n\
    sqr = (tmpSqr + tmpZp2) * e2InScale;\n\
\n\
    float mean, vari;\n\
    mean = sum * dimRatio;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3;\n\
    int2 coord_bias = (int2)(0, 0);\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_short8 src1, outval;\n\
    short zp = inputZP;\n\
    half4 tmpVal0, tmpVal1;\n\
    vxc_half8 dst;\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.x = coord.x;\n\
\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert3rdUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert4thUint8SubZpToFp32_4x4);\n\
        tmpData0 *= input_scale;\n\
        tmpData1 *= input_scale;\n\
        tmpData2 *= input_scale;\n\
        tmpData3 *= input_scale;\n\
\n\
        vxc_float4 norm;\n\
        tmpData0 -= mean;\n\
        norm = scale_f0 * vari * tmpData0 + bias_f0;\n\
\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
\n\
        tmpData1 -= mean;\n\
        norm = scale_f1 * vari * tmpData1 + bias_f1;\n\
\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            UniPackFP16even_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        coord_out.x = coord.x;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
\n\
        tmpData2 -= mean;\n\
        norm = scale_f0 * vari * tmpData2 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
\n\
        tmpData3 -= mean;\n\
        norm = scale_f1 * vari * tmpData3 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            UniPackFP16even_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        coord_out.x += 8;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void layer_norm_U8toF16_2D(\n\
    image2d_t input,\n\
    image2d_t bias,\n\
    image2d_t scale,\n\
    image2d_t output,\n\
        float eps)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), 0, 0);\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    vxc_int4 tmpSum1;\n\
    vxc_int4 tmpSqr1;\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
        tmpSum += (tmpSum1.x);\n\
        VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
        tmpSqr += (tmpSqr1.x + tmpZp1 * tmpSum1.x);\n\
    }\n\
    sum = (tmpSum + sumInZp) * input_scale;\n\
    sqr = (tmpSqr + tmpZp2) * e2InScale;\n\
\n\
    float mean, vari;\n\
    mean = sum * dimRatio;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    vari += eps;\n\
    vari = rsqrt(vari);\n\
    vxc_float4  tmpData0, tmpData1, tmpData2, tmpData3;\n\
    int2 coord_bias = (int2)(0, 0);\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_short8 src1, outval;\n\
    short zp = inputZP;\n\
    half4 tmpVal0, tmpVal1;\n\
    vxc_half8 dst;\n\
\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 16)\n\
    {\n\
        coord_bias.x = coord.x;\n\
        VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert2ndUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData2, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert3rdUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData3, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert4thUint8SubZpToFp32_4x4);\n\
        tmpData0 *= input_scale;\n\
        tmpData1 *= input_scale;\n\
        tmpData2 *= input_scale;\n\
        tmpData3 *= input_scale;\n\
\n\
        vxc_float4 norm;\n\
        tmpData0 -= mean;\n\
        norm = scale_f0 * vari * tmpData0 + bias_f0;\n\
\n\
        scale_f0 = read_imagef(scale, coord_bias);\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
\n\
        coord_bias.x += 4;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
\n\
        tmpData1 -= mean;\n\
        norm = scale_f1 * vari * tmpData1 + bias_f1;\n\
\n\
        scale_f1 = read_imagef(scale, coord_bias);\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            UniPackFP16even_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        coord_out.x = coord.x;\n\
        VXC_WriteImage(output, coord_out, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        tmpData2 -= mean;\n\
        norm = scale_f0 * vari * tmpData2 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
\n\
        tmpData3 -= mean;\n\
        norm = scale_f1 * vari * tmpData3 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            UniPackFP16even_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        coord_out.x += 8;\n\
        VXC_WriteImage(output, coord_out, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of layer_normalization_u8_f16_vx*/

static const char layer_normalization_wh_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform int width;\n\
\n\
_viv_uniform int height;\n\
\n\
_viv_uniform int height_depth;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_sumSqr_F16toF32(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0;\n\
        float sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_sumSqr_F16toF32_2D(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0;\n\
        float sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_F16toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    coord_para.z = (ushort)gidz / (ushort)(height_depth);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    vxc_half8 dst;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0),\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y;\n\
        coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 - mean_vari.s0;\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        sub = tmpData1 - mean_vari.s0;\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniConvertHalfToFp16_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_F16toF16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_t output, float eps)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_bias = (int2)(0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_bias);\n\
        coord_bias.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    coord_bias = coord;\n\
\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_short8 outval;\n\
    half4 tmpVal0, tmpVal1;\n\
    vxc_half8 dst;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.y = coord.y;\n\
        VXC_ReadImage(src1, scale, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 - mean_vari.s0;\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        sub = tmpData1 - mean_vari.s0;\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniConvertHalfToFp16_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_WriteImage(output, coord, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_F16toU8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    coord_para.z = (ushort)gidz / (ushort)(height_depth);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_uchar16 outval;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0),\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y;\n\
        coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 - mean_vari.s0;\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        sub = tmpData1 - mean_vari.s0;\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_F16toU8_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_t output, float eps)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_bias = (int2)(0, 0);\n\
    vxc_short8 src0;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h, in_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_bias);\n\
        coord_bias.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    coord_bias = coord;\n\
\n\
    vxc_float4  tmpData0, tmpData1;\n\
    vxc_uchar16 outval;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.y = coord.y;\n\
        VXC_ReadImage(src1, scale, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, in_h, src0, 16);\n\
        VXC_DP4x4(tmpData0, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(tmpData1, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertSecFp16Fp32_4x4);\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
\n\
        vxc_float4 sub, norm;\n\
        sub = tmpData0 - mean_vari.s0;\n\
        norm = scale_f0 * mean_vari.s1 * sub + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        sub = tmpData1 - mean_vari.s0;\n\
        norm = scale_f1 * mean_vari.s1 * sub + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_wh_f16_vx*/

static const char layer_normalization_wh_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniInt16SumSqr_dp8x2;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform int width;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform int height;\n\
\n\
_viv_uniform int height_depth;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform int inputZP;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_sumSqr_I16toF32(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    float4 tmpSumSqr = (float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            vxc_float4 sumsqr;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniInt16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
        tmpSumSqr.x *= input_scale;\n\
        tmpSumSqr.y *= e2InScale;\n\
    }\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        float4 data = (float4)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            data.x += dot(tmp_sum[i], one);\n\
            data.y += dot(tmp_sqr[i], one);\n\
        }\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_sumSqr_I16toF32_2D(\n\
    image2d_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_short8 src0;\n\
    float4 tmpSumSqr = (float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            vxc_float4 sumsqr;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniInt16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
        tmpSumSqr.x *= input_scale;\n\
        tmpSumSqr.y *= e2InScale;\n\
    }\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        float4 data = (float4)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            data.x += dot(tmp_sum[i], one);\n\
            data.y += dot(tmp_sqr[i], one);\n\
        }\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_I16toI16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    coord_para.z = (ushort)gidz / (ushort)(height_depth);\n\
    vxc_short8 src0, src1, outval;\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y;\n\
        coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean_vari.s0;\n\
        tmpData1 = tmpData1 * input_scale - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_I16toI16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_t output, float eps)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_bias = (int2)(0, 0);\n\
    vxc_short8 src0, src1, outval;\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_bias);\n\
        coord_bias.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    coord_bias = coord;\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.y = coord.y;\n\
        VXC_ReadImage(src1, scale, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean_vari.s0;\n\
        tmpData1 = tmpData1 * input_scale - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_wh_i16_vx*/

static const char layer_normalization_wh_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform int width;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform int height;\n\
\n\
_viv_uniform int height_depth;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform int group_num;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndUint8SubZpToFp32_4x4;\n\
_viv_uniform int inputZP;\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_sumSqr_U8toF32(\n\
    image2d_array_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.w, baseAddr_a);\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_OP4(img_load_3d, src0, input, coord.xywz, 0,\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1);\n\
        }\n\
        sqr += (tmpSqr * e2InScale + rowSumScale);\n\
        sum = (tmpSum + sumInZp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_sumSqr_U8toF32_2D(\n\
    image2d_t input, image2d_t output)\n\
{\n\
    int gidx = get_global_id(0) << 4;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_uchar16 src0;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, 0,\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1);\n\
            tmpSum += (tmpSum1);\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1);\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1);\n\
        }\n\
        sqr += (tmpSqr * e2InScale + rowSumScale);\n\
        sum = (tmpSum + sumInZp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(get_group_id(0) << 2, gidz);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 data = (float4)(sum, sqr, 0, 0);\n\
        write_imagef(output, coord_out, data);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_U8toF16(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    coord_para.z = (ushort)gidz / (ushort)(height_depth);\n\
    vxc_uchar16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y; coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean_vari.s0;\n\
        tmpData1 = tmpData1 * input_scale - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertHalfToFp16_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_U8toF16_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_t output, float eps)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_bias = (int2)(0, 0);\n\
    vxc_uchar16 src0;\n\
    vxc_short8 src1, outval;\n\
    vxc_half8 scale_h, dst;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_bias);\n\
        coord_bias.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    coord_bias = coord;\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    half4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, 0,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.y = coord.y;\n\
        VXC_ReadImage(src1, scale, coord, 0,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean_vari.s0;\n\
        tmpData1 = tmpData1 * input_scale - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        _viv_asm(CONV, tmpVal0, norm);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        _viv_asm(CONV, tmpVal1, norm);\n\
\n\
        VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertHalfToFp16_2x8);\n\
        _viv_asm(COPY, outval, dst, 16);\n\
        VXC_WriteImage(output, coord, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_U8toU8(\n\
    image2d_array_t input, image2d_array_t bias, image2d_array_t scale, image2d_t meanVari,\n\
    image2d_array_t output, float eps)\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int4 coord_in = (int4)(get_global_id(0), 0, gidz, gidz);\n\
    int2 coord_sum = (int2)(0, gidz);\n\
    int4 coord_para = coord;\n\
    coord_para.z = (ushort)gidz / (ushort)(height_depth);\n\
    vxc_uchar16 src0 , outval;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_sum);\n\
        coord_sum.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    int4 coord_bias = coord_para;\n\
\n\
    int8 input_desc, scale_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr_a = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.z, baseAddr_a);\n\
\n\
    _viv_asm(COPY, scale_desc, scale, sizeof(scale_desc));\n\
    int baseAddr_c = (int)coord_para.z * scale_desc.s4 + scale_desc.s0;\n\
    _viv_asm(MOV, coord_para.w, baseAddr_c);\n\
\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_OP4(img_load_3d, src0, input, coord_in, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y ++;\n\
        coord_para.y = coord.y;\n\
        coord_bias.y = coord.y;\n\
        VXC_OP4(img_load_3d, src1, scale, coord_para.xywz, 0,\n\
                        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean_vari.s0;\n\
        tmpData1 = tmpData1 * input_scale - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, outval, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layernorm_wh_U8toU8_2D(\n\
    image2d_t input, image2d_t bias, image2d_t scale, image2d_t meanVari,\n\
    image2d_t output, float eps)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_bias = (int2)(0, 0);\n\
    vxc_uchar16 src0, outval;\n\
    vxc_short8 src1;\n\
    vxc_half8 scale_h;\n\
    vxc_float4 bias_f0, bias_f1, scale_f0, scale_f1;\n\
    vxc_float4 mean_vari = (vxc_float4)(0);\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari += read_imagef(meanVari, coord_bias);\n\
        coord_bias.x += 4;\n\
    }\n\
    mean_vari *= dimRatio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    coord_bias = coord;\n\
\n\
    short zp = inputZP;\n\
    vxc_float4  tmpData0, tmpData1, norm;\n\
    vxc_int4 tmpVal0, tmpVal1;\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, 0,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_bias.y = coord.y;\n\
        VXC_ReadImage(src1, scale, coord, 0,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        bias_f0 = read_imagef(bias, coord_bias);\n\
        coord_bias.x += 4;\n\
        bias_f1 = read_imagef(bias, coord_bias);\n\
        coord_bias.x = coord.x;\n\
\n\
        _viv_asm(COPY, scale_h, src1, 16);\n\
        VXC_DP4x4(scale_f0, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                UniFP16toFP32Lo4_dp4x4);\n\
        VXC_DP4x4(scale_f1, scale_h, scale_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvertSecFp16Fp32_4x4);\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4);\n\
        VXC_DP4x4(tmpData1, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert2ndUint8SubZpToFp32_4x4);\n\
        tmpData0 = tmpData0 * input_scale - mean_vari.s0;\n\
        tmpData1 = tmpData1 * input_scale - mean_vari.s0;\n\
\n\
        norm = scale_f0 * mean_vari.s1 * tmpData0 + bias_f0;\n\
        tmpVal0 = convert_int4_rte(norm * outputScale + output_zp);\n\
        norm = scale_f1 * mean_vari.s1 * tmpData1 + bias_f1;\n\
        tmpVal1 = convert_int4_rte(norm * outputScale + output_zp);\n\
\n\
        VXC_DP2x8(outval, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniConvertInt32toUint8_2x8);\n\
        VXC_WriteImage(output, coord, outval, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of layer_normalization_wh_u8_vx*/

static const char log_softmax_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform int         inputWidthRemain4;\n\
_viv_uniform VXC_512Bits uniGetSubData0to3_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubData4to7_4x4;\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0(read_fun, vert_max_fun, horz_max_fun) \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        vert_max_fun(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        vert_max_fun(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16)); \\\n\
    horz_max_fun(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    horz_max_fun(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_float4 prob; \\\n\
    float fProbSum = 0; \\\n\
    const float4 one4 = (float4)(1.0, 1.0, 1.0, 1.0); \\\n\
    int idx = 0; \\\n\
    for (coord.x = 0; coord.x < inputWidth; idx ++) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
        prob *= scaleLogE; \\\n\
        prob = exp2(prob); \\\n\
        fProbSum += dot(prob, one4); \\\n\
        coord.x += 4; \\\n\
    } \\\n\
    read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, img_val0, val0, 16); \\\n\
    VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
    prob *= scaleLogE; \\\n\
    if(inputWidthRemain4 == 1) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.yzw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 2) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.zw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 3) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.z = exp2(prob.z); \\\n\
        prob.w = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    vxc_float4 probSum_log; \\\n\
    probSum_log.x = log2(fProbSum) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        dst_type vec0, vec1; \\\n\
        save_type dst; \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        prob = prob * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, vec0, prob); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        prob = prob * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, vec1, prob); \\\n\
        VXC_DP2x8(dst, vec0, vec1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
        write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 8; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
                        save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                 CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage, VXC_WriteImage); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_TOF32_SAVE(read_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        write_imagef(output, coord, prob); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(prob, img_val0, val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4); \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        write_imagef(output, coord, prob); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_AXIS0_TOF32(src_name, src_type, copy_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_TOF32_SAVE(VXC_ReadImage2DArray) \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0_TOF32(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_TOF32(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS0_TOF32_2D(src_name, src_type, copy_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void log_softmax_axis0_##src_name##toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_t       output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    LOGSOFTMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS0_TOF32_SAVE(VXC_ReadImage) \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS0_TOF32_2D(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
LOGSOFTMAX_AXIS0_TOF32_2D(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32_2D(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
LOGSOFTMAX_AXIS0_TOF32_2D(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
"; /* end of log_softmax_axis0_vx*/

static const char log_softmax_axis0_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform int         inputWidthRemain4;\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16(read_fun) \\\n\
    vxc_half8 img_val0, img_val1, img_val2, img_val3; \\\n\
    vxc_short8 val0, val1, val2, val3; \\\n\
    vxc_half8 val; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        VXC_VertMax3_Half(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_VertMax3_Half(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16)); \\\n\
    VXC_HorzMax3_Half(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    VXC_HorzMax3_Half(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_ushort8   bf_val_tmp; \\\n\
    vxc_float4 vecA; \\\n\
    _viv_asm(COPY, bf_val_tmp, val, 16); \\\n\
    VXC_DP2x8(bf_val_tmp, bf_val_tmp, zero,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, bf_val_tmp, 16); \\\n\
    vxc_float4 prob; \\\n\
    float fProbSum = 0; \\\n\
    const float4 one4 = (float4)(1.0, 1.0, 1.0, 1.0); \\\n\
    float max_value = vecA.x * scaleLogE; \\\n\
    float max_value_orig = vecA.x; \\\n\
    for (coord.x = 0; coord.x < inputWidth; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob * scaleLogE - max_value; \\\n\
        prob = exp2(prob); \\\n\
        fProbSum += dot(prob, one4); \\\n\
        coord.x += 4; \\\n\
    } \\\n\
    read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
    prob = prob * scaleLogE - max_value; \\\n\
    if(inputWidthRemain4 == 1) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.yzw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 2) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.zw = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    else if(inputWidthRemain4 == 3) \\\n\
    { \\\n\
        prob.x = exp2(prob.x); \\\n\
        prob.y = exp2(prob.y); \\\n\
        prob.z = exp2(prob.z); \\\n\
        prob.w = 0; \\\n\
        fProbSum += dot(prob, one4); \\\n\
    } \\\n\
    vxc_float4 probSum_log; \\\n\
    probSum_log.x = log2(fProbSum) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16TOBF16_SAVE(read_fun, write_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob - max_value_orig; \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        vxc_ushort8 tmp, dst; \\\n\
        _viv_asm(COPY, tmp, prob, 16); \\\n\
        dst.s0123 = tmp.s1357; \\\n\
        write_fun(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16TOF16_SAVE(read_fun, write_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob - max_value_orig; \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        half4 vec; \\\n\
        vxc_half4 tmp; \\\n\
        vxc_short4 dst; \\\n\
        _viv_asm(CONV, vec, prob); \\\n\
        VXC_DP4x4(tmp, vec, vec, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
        _viv_asm(COPY, dst, tmp, 8); \\\n\
        write_fun(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS0_BF16TOF32_SAVE(read_fun) \\\n\
    for (coord.x = 0; coord.x < axisSize; ) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(bf_val_tmp, val0, zero,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, prob, bf_val_tmp, 16); \\\n\
        prob = prob - max_value_orig; \\\n\
        prob = prob * betaValue - probSum_log.xxxx; \\\n\
        write_imagef(output, coord, prob); \\\n\
        coord.x += 4; \\\n\
    }\n\
\n\
__kernel void log_softmax_axis0_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOBF16_SAVE(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF16_SAVE(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0);\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF32_SAVE(VXC_ReadImage2DArray)\n\
}\n\
__kernel void log_softmax_axis0_BF16toBF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(16, get_global_id(0));\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOBF16_SAVE(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(16, get_global_id(0));\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF16_SAVE(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
__kernel void log_softmax_axis0_BF16toF32_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t        output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(16, get_global_id(0));\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16(VXC_ReadImage)\n\
    LOGSOFTMAX_PROCESS_AXIS0_BF16TOF32_SAVE(VXC_ReadImage)\n\
}\n\
\n\
"; /* end of log_softmax_axis0_BF16_vx*/

static const char log_softmax_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGetSubLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubHiData_4x4;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS1(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
    coord.y = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum0 += data0; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum1 += data0; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
                 OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    coord.y = 0; \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst0, data0); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst1, data0); \\\n\
        VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
        write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define LOGSOFTMAX_AXIS1(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
\n\
\n\
LOGSOFTMAX_AXIS1(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half)\n\
\n\
LOGSOFTMAX_AXIS1(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8, CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS1_2D(src_name, dst_name, src_type,\\\n\
copy_type, dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage, VXC_WriteImage); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS1_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half)\n\
\n\
LOGSOFTMAX_AXIS1_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4,\\\n\
vxc_uchar8, CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8, CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
\n\
#define LOGSOFTMAX_AXIS1_TOF32(src_name, src_type, copy_type, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    coord.y = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x -= 4; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS1_TOF32(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_TOF32(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer)\n\
\n\
#define LOGSOFTMAX_AXIS1_TOF32_2D(src_name, src_type, copy_type, vert_max_fun) \\\n\
__kernel void log_softmax_axis1_##src_name##toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_t       output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    coord.y = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x -= 4; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS1_TOF32_2D(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS1_TOF32_2D(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32_2D(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS1_TOF32_2D(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer)\n\
"; /* end of log_softmax_axis1_vx*/

static const char log_softmax_axis1_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS1_BF16(read_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_VertMax3_Half(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize); \\\n\
    _viv_asm(COPY, tmp0, max, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, max_lo, tmp1, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, max_hi, tmp1, 16); \\\n\
    coord.y = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0 = data0 - max_lo; \\\n\
        data0 *= scaleLogE; \\\n\
        sum0  += exp2(data0); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0 = data0 - max_hi; \\\n\
        data0 *= scaleLogE; \\\n\
        sum1  += exp2(data0); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while (coord.y < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
__kernel void log_softmax_axis1_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(COPY, dst0, data0, 16);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(COPY, dst1, data0, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    half4 dst0, dst1;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(CONV, dst0, data0);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(CONV, dst1, data0);\n\
        VXC_DP2x8(vec0, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
        vxc_short8 vect;\n\
        _viv_asm(COPY, vect, vec0, 16);\n\
        VXC_WriteImage2DArray(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.y = 0;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        write_imagef(output, coord, data0);\n\
        coord.x += 4;\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        write_imagef(output, coord, data0);\n\
        coord.x -= 4;\n\
        coord.y++;\n\
    }\n\
    while (coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toBF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
\n\
    coord.y = 0;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    do\n\
    {\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(COPY, dst0, data0, 16);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(COPY, dst1, data0, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF16_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
\n\
    coord.y = 0;\n\
    half4 dst0, dst1;\n\
    do\n\
    {\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(CONV, dst0, data0);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(CONV, dst1, data0);\n\
        VXC_DP2x8(vec0, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
        vxc_short8 vect;\n\
        _viv_asm(COPY, vect, vec0, 16);\n\
        VXC_WriteImage(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
    }\n\
    while(coord.y < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis1_BF16toF32_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t        output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
\n\
    coord.y = 0;\n\
    do\n\
    {\n\
        VXC_ReadImage(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        write_imagef(output, coord, data0);\n\
        coord.x += 4;\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        write_imagef(output, coord, data0);\n\
        coord.x -= 4;\n\
        coord.y++;\n\
    }\n\
    while (coord.y < axisSize);\n\
}\n\
"; /* end of log_softmax_axis1_BF16_vx*/

static const char log_softmax_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       rlogE;\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       betaValue;\n\
_viv_uniform float       scaleLogE;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGetSubLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubHiData_4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS2(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
    coord.z = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum0 += data0; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 *= scaleLogE; \\\n\
        data0 = exp2(data0); \\\n\
        sum1 += data0; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS2_SAVE(dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, read_fun, write_fun) \\\n\
    coord.z = 0; \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst0, data0); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        data0 = data0 * OUT_SCALE + OUT_OFFSET; \\\n\
        _viv_asm(conv_mode, dst1, data0); \\\n\
        VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
        write_fun(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
#define LOGSOFTMAX_AXIS2(src_name, dst_name, src_type, copy_type,\\\n\
dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, vert_max_fun) \\\n\
__kernel void log_softmax_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    LOGSOFTMAX_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_ReadImage2DArray, VXC_WriteImage2DArray); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS2(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Half)\n\
\n\
LOGSOFTMAX_AXIS2(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS2(I8, I8,  vxc_char16, vxc_char16, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
LOGSOFTMAX_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, VXC_VertMax3_Integer)\n\
\n\
\n\
#define LOGSOFTMAX_AXIS2_TOF32(src_name, src_type, copy_type, vert_max_fun) \\\n\
__kernel void log_softmax_axis2_##src_name##toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    float input_Scale, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    vxc_float4 sum0, sum1; \\\n\
    LOGSOFTMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    coord.z = 0; \\\n\
    do \\\n\
    { \\\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubLoData_4x4); \\\n\
        data0 = data0 * betaValue - sum0; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x += 4; \\\n\
        VXC_DP4x4(data0, vec0, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubHiData_4x4); \\\n\
        data0 = data0 * betaValue - sum1; \\\n\
        write_imagef(output, coord, data0); \\\n\
        coord.x -= 4; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
}\n\
\n\
LOGSOFTMAX_AXIS2_TOF32(F16, vxc_half8,   vxc_short8, VXC_VertMax3_Half)\n\
LOGSOFTMAX_AXIS2_TOF32(I16, vxc_short8,  vxc_short8, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2_TOF32(I8,  vxc_char16,  vxc_char16, VXC_VertMax3_Integer)\n\
LOGSOFTMAX_AXIS2_TOF32(U8,  vxc_uchar16, vxc_uchar16, VXC_VertMax3_Integer)\n\
\n\
#define LOGSOFTMAX_PROCESS_AXIS2_BF16(read_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_VertMax3_Half(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize); \\\n\
    _viv_asm(COPY, tmp0, max, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, max_lo, tmp1, 16); \\\n\
    VXC_DP2x8(tmp1, tmp0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, max_hi, tmp1, 16); \\\n\
    coord.z = 0; \\\n\
    sum0 = 0; \\\n\
    sum1 = 0; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0  = data0 - max_lo; \\\n\
        data0 *= scaleLogE; \\\n\
        sum0  += exp2(data0); \\\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, data0, tmp1, 16); \\\n\
        data0  = data0 - max_hi; \\\n\
        data0 *= scaleLogE; \\\n\
        sum1  += exp2(data0); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while (coord.z < axisSize); \\\n\
    sum0 = log2(sum0) * rlogE; \\\n\
    sum1 = log2(sum1) * rlogE;\n\
\n\
__kernel void log_softmax_axis2_BF16toBF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.z = 0;\n\
    vxc_ushort8 dst0, dst1, dst;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(COPY, dst0, data0, 16);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(COPY, dst1, data0, 16);\n\
        VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
    while(coord.z < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis2_BF16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.z = 0;\n\
    half4 dst0, dst1;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        _viv_asm(CONV, dst0, data0);\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        _viv_asm(CONV, dst1, data0);\n\
        VXC_DP2x8(vec0, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
        vxc_short8 vect;\n\
        _viv_asm(COPY, vect, vec0, 16);\n\
        VXC_WriteImage2DArray(output, coord, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.z++;\n\
    }\n\
    while(coord.z < axisSize);\n\
}\n\
\n\
__kernel void log_softmax_axis2_BF16toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output,\n\
    float input_Scale,\n\
    int   axisVal )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_short8 in0;\n\
    vxc_half8 vec0, max;\n\
    vxc_float4 data0;\n\
    vxc_float4 sum0, sum1;\n\
    vxc_float4 max_lo, max_hi;\n\
    vxc_ushort8   tmp0, tmp1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
\n\
    LOGSOFTMAX_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
\n\
    coord.z = 0;\n\
    do\n\
    {\n\
        VXC_ReadImage2DArray(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_lo;\n\
        data0 = data0 * betaValue - sum0;\n\
        write_imagef(output, coord, data0);\n\
        coord.x += 4;\n\
        VXC_DP2x8(tmp1, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, data0, tmp1, 16);\n\
        data0 = data0 - max_hi;\n\
        data0 = data0 * betaValue - sum1;\n\
        write_imagef(output, coord, data0);\n\
        coord.x -= 4;\n\
        coord.z++;\n\
    }\n\
    while (coord.z < axisSize);\n\
}\n\
"; /* end of log_softmax_axis2_vx*/

static const char logical_not_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void logical_not_I8toI8(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_char8 src0;\n\
    vxc_char8 dst;\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    dst = !src0;\n\
    dst *= (-1);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void logical_not_I8toI8_2D(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char8 src0;\n\
    vxc_char8 dst;\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    dst = !src0;\n\
    dst *= (-1);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of logical_not_vx*/

static const char logical_ops_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define TENSORLOGICAL_PROCESS(input_type, copy_type, output_type, out_copy_type,\\\n\
lgc_op, lgc_op2, read_fun, write_fun) \\\n\
    input_type vA;\\\n\
    copy_type  src0;\\\n\
    input_type vB;\\\n\
    copy_type  src1;\\\n\
    read_fun(vA,in0,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src0, vA, 16); \\\n\
    read_fun(vB,in1,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src1, vB, 16); \\\n\
    output_type dst; \\\n\
    dst = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    dst *= (-1); \\\n\
    out_copy_type data; \\\n\
    _viv_asm(COPY, data, dst, 16); \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
\n\
#define TENSORLOGICAL(name0, src_type_name, dst_type_name, input_type, copy_type,\\\n\
output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\\\n\
    TENSORLOGICAL_PROCESS(input_type, copy_type, output_type, out_copy_type,\\\n\
    lgc_op, lgc_op2, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
#define TENSORLOGICAL_2D(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    TENSORLOGICAL_PROCESS(input_type, copy_type, output_type, out_copy_type,\\\n\
    lgc_op, lgc_op2, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniMulShortMinus1toFp16_2x8;\n\
\n\
#define TENSORLOGICAL_FP_PROCESS(input_type, copy_type, output_type,\\\n\
out_copy_type, lgc_op, lgc_op2,  read_fun, write_fun) \\\n\
    input_type vA;\\\n\
    copy_type  src0;\\\n\
    input_type vB;\\\n\
    copy_type  src1;\\\n\
    read_fun(vA,in0,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src0, vA, 16); \\\n\
    read_fun(vB,in1,coord,VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0,7,0,VXC_RM_TowardZero,0));\\\n\
    _viv_asm(COPY, src1, vB, 16); \\\n\
    output_type dst; \\\n\
    dst = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    vxc_half8 tmpOut; \\\n\
    VXC_DP2x8(tmpOut,dst,dst,VXC_MODIFIER(0,7,0,VXC_RM_TowardZero, 0),uniMulShortMinus1toFp16_2x8); \\\n\
    out_copy_type data; \\\n\
    _viv_asm(COPY, data, tmpOut, 16); \\\n\
    write_fun(output, coord, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
\n\
\n\
#define TENSORLOGICAL_FP(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\\\n\
    TENSORLOGICAL_FP_PROCESS(input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2,\\\n\
    VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
#define TENSORLOGICAL_FP_2D(name0, src_type_name, dst_type_name, input_type,\\\n\
copy_type, output_type, out_copy_type, lgc_op, lgc_op2) \\\n\
    __kernel void logical_##name0##_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t in0, \\\n\
    __read_only  image2d_array_t in1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    TENSORLOGICAL_FP_PROCESS(input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2,\\\n\
    VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
//          name0, src_name, dst_name, input_type, copy_type, output_type, out_copy_type, lgc_op, lgc_op2\n\
TENSORLOGICAL(or,      I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ||, )\n\
//TENSORLOGICAL(or,      U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ||, )\n\
//TENSORLOGICAL(or,      I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
//TENSORLOGICAL_FP(or,   F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
TENSORLOGICAL(and,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   &&, )\n\
//TENSORLOGICAL(and,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  &&, )\n\
//TENSORLOGICAL(and,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
//TENSORLOGICAL_FP(and,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
TENSORLOGICAL(xor,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ^, !!)\n\
//TENSORLOGICAL(xor,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ^, !!)\n\
//TENSORLOGICAL(xor,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
//TENSORLOGICAL_FP(xor,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
\n\
TENSORLOGICAL_2D(or,      I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ||, )\n\
//TENSORLOGICAL_2D(or,      U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ||, )\n\
//TENSORLOGICAL_2D(or,      I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
//TENSORLOGICAL_FP_2D(or,   F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ||, )\n\
TENSORLOGICAL_2D(and,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   &&, )\n\
//TENSORLOGICAL_2D(and,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  &&, )\n\
//TENSORLOGICAL_2D(and,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
//TENSORLOGICAL_FP_2D(and,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  &&, )\n\
TENSORLOGICAL_2D(xor,     I8,     I8,     vxc_char8,   vxc_char8,   vxc_char8,   vxc_char8,   ^, !!)\n\
//TENSORLOGICAL_2D(xor,     U8,     U8,     vxc_uchar8,  vxc_uchar8,  vxc_char8,   vxc_uchar8,  ^, !!)\n\
//TENSORLOGICAL_2D(xor,     I16,    I16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
//TENSORLOGICAL_FP_2D(xor,  F16,    F16,    vxc_short8,  vxc_short8,  vxc_short8,  vxc_short8,  ^, !!)\n\
"; /* end of logical_ops_vx*/

static const char lstmunit_activation_BP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_BP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_BP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_F16_vx*/

static const char lstmunit_activation_BP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_BP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_BP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_BP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_BP_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_BP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_BP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_U8_vx*/

static const char lstmunit_activation_B_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_B_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_B_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_B_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_B_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_B_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_B_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_B_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_B_F16_vx*/

static const char lstmunit_activation_B_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_B_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
LSTMUNIT_B_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_B_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_B_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx + b0; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_B_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_B_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_B_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_B_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_B_U8_vx*/

static const char lstmunit_activation_CBP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CBP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CBP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_F16_vx*/

static const char lstmunit_activation_CBP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CBP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CBP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CBP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CBP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_U8_vx*/

static const char lstmunit_activation_CB_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CB_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CB_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_F16_vx*/

static const char lstmunit_activation_CB_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CB_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CB_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy + b1; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz + b2; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CB_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CB_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CB_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CB_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_U8_vx*/

static const char lstmunit_activation_CLP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
\n\
#define LSTMUNIT_CLP_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CLP_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CLP_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CLP_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CLP_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
#define LSTMUNIT_CLP_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CLP_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CLP_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CLP_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CLP_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CLP_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CLP_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CLP_F16_vx*/

static const char lstmunit_activation_CL_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
#define LSTMUNIT_CL_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CL_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CL_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CL_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CL_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CL_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CL_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CL_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CL_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CL_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, b0, b1, b2; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_f, coord_in.xw); \\\n\
    b1 = read_imagef(bias_c, coord_in.xw); \\\n\
    b2 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_f_t = data_f_t * w0 + b0; \\\n\
    data_g_t = data_g_t * w1 + b1; \\\n\
    data_o_t = data_o_t * w2 + b2; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CL_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CL_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CL_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CL_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CL_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CL_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CL_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CL_F16_vx*/

static const char lstmunit_activation_CSP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CSP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CSP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_F16_vx*/

static const char lstmunit_activation_CSP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CSP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CSP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CSP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CSP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_U8_vx*/

static const char lstmunit_activation_CS_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_CS_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CS_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CS_F16_vx*/

static const char lstmunit_activation_CS_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_CS_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_CS_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = 1.0 - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_CS_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_CS_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_CS_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_CS_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CS_U8_vx*/

static const char lstmunit_activation_LP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
#define LSTMUNIT_LP_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_LP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_LP_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_LP_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_LP_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_LP_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_LP_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_LP_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_LP_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_LP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_LP_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_LP_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_LP_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_LP_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_LP_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_LP_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_LP_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_LP_F16_vx*/

static const char lstmunit_activation_L_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
\n\
#define LSTMUNIT_L_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_L_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_L_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_L_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_L_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_L_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_L_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_L_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_L_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_L_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 w0, w1, w2, w3, b0, b1, b2, b3; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    data_i_t = data_i_t * w0 + b0; \\\n\
    data_f_t = data_f_t * w1 + b1; \\\n\
    data_g_t = data_g_t * w2 + b2; \\\n\
    data_o_t = data_o_t * w3 + b3; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 cell_data; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, cell_data, data_c_t); \\\n\
    VXC_DP4x4(src0, cell_data, cell_data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_L_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_L_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_L_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_L_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_L_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_L_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_L_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
"; /* end of lstmunit_activation_L_F16_vx*/

static const char lstmunit_activation_SP_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_SP_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_SP_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_F16_vx*/

static const char lstmunit_activation_SP_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_SP_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_U8_FP32(U8,  SIGMOID, int4, vxc_uchar4,  vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_SP_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_SP_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_SP_U8_FP16(U8,  SIGMOID, int4, vxc_uchar4,  vxc_uchar4, sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_SP_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_SP_U8_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_U8_vx*/

static const char lstmunit_activation_S_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniFp16AddFp16toFp32_4x4;\n\
\n\
#define LSTMUNIT_S_FP16_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_F16to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3; \\\n\
    vxc_half8  src0, src1, src2, src3; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_FP16_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_S_FP16_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP32(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_S_FP16_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_F16to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0, vect1, vect2, vect3, vect4; \\\n\
    vxc_half8  src0, src1, src2, src3, src4; \\\n\
    vxc_short8 vect10, vect11, vect12, vect13; \\\n\
    vxc_half8  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    VXC_ReadImage(vect0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src0, vect0, 16); \\\n\
    VXC_ReadImage(vect10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src10, vect10, 16); \\\n\
    VXC_ReadImage(vect1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, vect1, 16); \\\n\
    VXC_ReadImage(vect11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src11, vect11, 16); \\\n\
    VXC_ReadImage(vect2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src2, vect2, 16); \\\n\
    VXC_ReadImage(vect12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src12, vect12, 16); \\\n\
    VXC_ReadImage(vect3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src3, vect3, 16); \\\n\
    VXC_ReadImage(vect13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src13, vect13, 16); \\\n\
    VXC_ReadImage(vect4, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect4, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_i_t, src0, src10, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_f_t, src1, src11, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_g_t, src2, src12, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(data_o_t, src3, src13, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16AddFp16toFp32_4x4); \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src0, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src0, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_FP16_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I8,  SIGMOID, int4,  vxc_char4,  vxc_char4,  sigmoid)\n\
LSTMUNIT_S_FP16_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I16, SIGMOID, int4,  vxc_short4, vxc_short4, sigmoid)\n\
LSTMUNIT_S_FP16_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I8,  HARD_SIGMOID, int4,  vxc_char4,  vxc_char4,  hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_FP16_FP16(I16, HARD_SIGMOID, int4,  vxc_short4, vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_F16_vx*/

static const char lstmunit_activation_S_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
_viv_uniform float twoLogE;\n\
_viv_uniform float forget_bias;\n\
float4 sigmoid(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform float4 clip_Min_F;\n\
_viv_uniform float4 clip_Max_F;\n\
_viv_uniform VXC_512Bits uniExtractHalf4_4x4;\n\
_viv_uniform VXC_512Bits uniU8AddS32_4x4;\n\
_viv_uniform int4 input0Array_ZP;\n\
_viv_uniform int4 input1Array_ZP;\n\
_viv_uniform float4 input0Array_Scale;\n\
_viv_uniform float4 input1Array_Scale;\n\
\n\
#define LSTMUNIT_S_U8_FP32(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_U8to##out_type_name##_F32_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.zy); \\\n\
 \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_U8_FP32(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_U8_FP32(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_U8_FP32(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_U8_FP32(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
\n\
#define LSTMUNIT_S_U8_FP16(out_type_name, act_name, convert_type, dst_type, copy_type, act_func) \\\n\
__kernel void lstmunit_activation_S_U8to##out_type_name##_F16_##act_name( \\\n\
    __read_only  image2d_array_t  input_i_conv, \\\n\
    __read_only  image2d_array_t  input_f_conv, \\\n\
    __read_only  image2d_array_t  input_c_conv, \\\n\
    __read_only  image2d_array_t  input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_array_t  hstate_i_conv, \\\n\
    __read_only  image2d_array_t  hstate_f_conv, \\\n\
    __read_only  image2d_array_t  hstate_c_conv, \\\n\
    __read_only  image2d_array_t  hstate_o_conv, \\\n\
    __write_only image2d_array_t  output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    int _is_ln, int _is_cifg, int _is_hybrid, float cell_clip, float forgetBias \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    vxc_short8 vect0; \\\n\
    vxc_half8  src4; \\\n\
    vxc_uchar4 src0,  src1,  src2,  src3; \\\n\
    vxc_uchar4 src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 vecA, vecB; \\\n\
    VXC_ReadImage(src0, input_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src10, hstate_i_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src11, hstate_f_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src2, input_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src12, hstate_c_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src3, input_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src13, hstate_o_conv, coord_in.xy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(vect0, cell_state_in, coord_in.zy, 0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src4, vect0, 16); \\\n\
 \\\n\
    VXC_DP4x4(data_c_t, src4, src4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4); \\\n\
    VXC_DP4x4(vecA, src0, input0Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src10, input1Array_ZP.xxxx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_i_t = vecA * input0Array_Scale.xxxx + vecB * input1Array_Scale.xxxx; \\\n\
    VXC_DP4x4(vecA, src1, input0Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src11, input1Array_ZP.yyyy, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_f_t = vecA * input0Array_Scale.yyyy + vecB * input1Array_Scale.yyyy; \\\n\
    VXC_DP4x4(vecA, src2, input0Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src12, input1Array_ZP.zzzz, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_g_t = vecA * input0Array_Scale.zzzz + vecB * input1Array_Scale.zzzz; \\\n\
    VXC_DP4x4(vecA, src3, input0Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    VXC_DP4x4(vecB, src13, input1Array_ZP.wwww, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniU8AddS32_4x4);\\\n\
    data_o_t = vecA * input0Array_Scale.wwww + vecB * input1Array_Scale.wwww; \\\n\
 \\\n\
    convert_type dst0; \\\n\
    half4 dst_cell; \\\n\
    data_i_t = act_func(data_i_t); \\\n\
    data_f_t = act_func(data_f_t + forget_bias); \\\n\
    data_g_t = tangentH(data_g_t); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    _viv_asm(CONV, dst_cell, data_c_t); \\\n\
    VXC_DP4x4(src4, dst_cell, dst_cell, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractHalf4_4x4); \\\n\
    _viv_asm(COPY, vect0, src4, 8); \\\n\
    VXC_WriteImage(cell_state_out, coord_in.zy, vect0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    data_c_t = tangentH(data_c_t); \\\n\
    data_o_t = data_o_t * data_c_t * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, data_o_t); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 16); \\\n\
    VXC_WriteImage(output, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(h_state_out, coord_in.zy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
LSTMUNIT_S_U8_FP16(U8,  SIGMOID, int4,  vxc_uchar4, vxc_uchar4, sigmoid)\n\
LSTMUNIT_S_U8_FP16(F16, SIGMOID, half4, vxc_half4,  vxc_short4, sigmoid)\n\
LSTMUNIT_S_U8_FP16(U8,  HARD_SIGMOID, int4,  vxc_uchar4, vxc_uchar4, hard_sigmoid)\n\
LSTMUNIT_S_U8_FP16(F16, HARD_SIGMOID, half4, vxc_half4,  vxc_short4, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_U8_vx*/

static const char matrixmul_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Hi_4x4;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
\n\
#if (VX_VERSION==2)\n\
__kernel void gemm_F16F16toF16(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB,\n\
            int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    half4 valC;\n\
    vxc_short8 srcA0, srcA1, srcA2, srcA3, outC;\n\
    vxc_half8 tmpA0, tmpA1, tmpA2, tmpA3;\n\
    vxc_short16 srcB;\n\
    vxc_half16 tmpB;\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 4;\n\
        coord_b.y += 4;\n\
        _viv_asm(COPY, tmpA0, srcA0, 16);\n\
        _viv_asm(COPY, tmpA1, srcA1, 16);\n\
        _viv_asm(COPY, tmpA2, srcA2, 16);\n\
        _viv_asm(COPY, tmpA3, srcA3, 16);\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16);\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16);\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, tmpA2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, tmpA3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
                    uniGemmU8F16toF32Lo_4x4b);\n\
        sum0 += (tempA0);\n\
        sum1 += (tempA1);\n\
        sum2 += (tempA2);\n\
        sum3 += (tempA3);\n\
    }\n\
    coord_b.y = gidy;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(CONV, valC, sum0);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum1);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum2);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum3);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
#else\n\
__kernel void gemm_F16F16toF16(image2d_array_t inputA,\n\
            image2d_array_t inputB, image2d_array_t output,\n\
            int transposeA, int transposeB,\n\
            int adjointA, int adjointB, uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    half4 valC;\n\
    vxc_short8 srcA0, srcB0, srcA1, srcB1, outC;\n\
    vxc_half8 tmpA0, tmpB0, tmpA1, tmpB1;\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 4;\n\
        coord_b.y += 4;\n\
        _viv_asm(COPY, tmpA0, srcA0, 16);\n\
        _viv_asm(COPY, tmpB0, srcB0, 16);\n\
        _viv_asm(COPY, tmpA1, srcA1, 16);\n\
        _viv_asm(COPY, tmpB1, srcB1, 16);\n\
\n\
        VXC_DP4x4(tempA0, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4);\n\
        VXC_DP4x4(tempB0, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4);\n\
        VXC_DP4x4(tempA1, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4);\n\
        VXC_DP4x4(tempB1, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4);\n\
        VXC_DP4x4(tempA2, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4);\n\
        VXC_DP4x4(tempB2, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4);\n\
        VXC_DP4x4(tempA3, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4);\n\
        VXC_DP4x4(tempB3, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4);\n\
        sum0 += (tempA0 + tempB0);\n\
        sum1 += (tempA1 + tempB1);\n\
        sum2 += (tempA2 + tempB2);\n\
        sum3 += (tempA3 + tempB3);\n\
    }\n\
    coord_b.y = gidy;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(CONV, valC, sum0);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum1);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum2);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum3);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
#endif\n\
\n\
__kernel void gemm_F32F32toF32(image2d_array_t inputA,\n\
                               image2d_array_t inputB,\n\
                               image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    uint gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(gidx, 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    vxc_uchar16 outC;\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8);\n\
\n\
    for(int i = 0; i < K; i+=4)\n\
    {\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3;\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3;\n\
\n\
        coord_a.x = i;\n\
        coord_a.y = gidy;\n\
\n\
        coord_b.x = gidx;\n\
        coord_b.y = i;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        coord_a.y++;\n\
        tempA1 = read_imagef(inputA, coord_a);\n\
        coord_a.y++;\n\
        tempA2 = read_imagef(inputA, coord_a);\n\
        coord_a.y++;\n\
        tempA3 = read_imagef(inputA, coord_a);\n\
\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_b.y++;\n\
        tempB1 = read_imagef(inputB, coord_b);\n\
        coord_b.y++;\n\
        tempB2 = read_imagef(inputB, coord_b);\n\
        coord_b.y++;\n\
        tempB3 = read_imagef(inputB, coord_b);\n\
\n\
        sum0 += (tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3);\n\
        sum1 += (tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3);\n\
        sum2 += (tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3);\n\
        sum3 += (tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3);\n\
    }\n\
    coord_b = (int4)(gidx, gidy, get_global_id(2), 0);\n\
    write_imagef(output, coord_b, sum0);\n\
    coord_b.y++;\n\
    write_imagef(output, coord_b, sum1);\n\
    coord_b.y++;\n\
    write_imagef(output, coord_b, sum2);\n\
    coord_b.y++;\n\
    write_imagef(output, coord_b, sum3);\n\
}"; /* end of matrixmul_f16_vx*/

static const char matrixmul_f16f16_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row0Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16toFp32Row1Hi_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_F16_TO_QINT(dst_type_name, write_type) \\\n\
__kernel void gemm_F16F16to##dst_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1, srcA2, srcA3; \\\n\
    vxc_half8 tmpA0, tmpA1, tmpA2, tmpA3; \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        _viv_asm(COPY, tmpA2, srcA2, 16); \\\n\
        _viv_asm(COPY, tmpA3, srcA3, 16); \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, tmpA2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, tmpA3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        sum0 += (tempA0); \\\n\
        sum1 += (tempA1); \\\n\
        sum2 += (tempA2); \\\n\
        sum3 += (tempA3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    write_type outC; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#else\n\
#define GEMM_F16_TO_QINT(dst_type_name, write_type) \\\n\
__kernel void gemm_F16F16to##dst_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcB0, srcA1, srcB1; \\\n\
    vxc_half8 tmpA0, tmpB0, tmpA1, tmpB1; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpB0, srcB0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        _viv_asm(COPY, tmpB1, srcB1, 16); \\\n\
 \\\n\
        VXC_DP4x4(tempA0, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4); \\\n\
        VXC_DP4x4(tempB0, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4); \\\n\
        VXC_DP4x4(tempB1, tmpA0, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Lo_4x4); \\\n\
        VXC_DP4x4(tempB2, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row0Hi_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, tmpB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Lo_4x4); \\\n\
        VXC_DP4x4(tempB3, tmpA1, tmpB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmFp16toFp32Row1Hi_4x4); \\\n\
        sum0 += (tempA0 + tempB0); \\\n\
        sum1 += (tempA1 + tempB1); \\\n\
        sum2 += (tempA2 + tempB2); \\\n\
        sum3 += (tempA3 + tempB3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    write_type outC; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#endif\n\
GEMM_F16_TO_QINT(U8, vxc_uchar16)\n\
GEMM_F16_TO_QINT(I8, vxc_char16)\n\
GEMM_F16_TO_QINT(I16, vxc_short8)\n\
"; /* end of matrixmul_f16f16_u8_vx*/

static const char matrixmul_f16i16_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32A_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32B_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32C_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32D_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32Lo_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmF16I16toF32Hi_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmFp16I16MulZptoFp32_4x4;\n\
_viv_uniform float in1outScale;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_F16_QINT16_TO_QINT16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toI16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1, outC; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    vxc_short16 srcB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4_b(tempA0, srcB.hi, srcB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, srcB.hi, srcB.lo, tmpA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Hi_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, srcB.hi, srcB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, srcB.hi, srcB.lo, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmF16I16toF32Hi_4x4b); \\\n\
        VXC_DP4x4(tmpZpScale, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16I16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tmpZpScale.xxxx; \\\n\
        sum1 += tempA1 + tmpZpScale.yyyy; \\\n\
        sum2 += tempA2 + tmpZpScale.zzzz; \\\n\
        sum3 += tempA3 + tmpZpScale.wwww; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#else\n\
#define GEMM_F16_QINT16_TO_QINT16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toI16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1, outC; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    vxc_short8 srcB0, srcB1; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA0, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32A_4x4); \\\n\
        VXC_DP4x4(tempB0, tmpA0, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32B_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32C_4x4); \\\n\
        VXC_DP4x4(tempB1, tmpA0, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32D_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32A_4x4); \\\n\
        VXC_DP4x4(tempB2, tmpA1, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32B_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, srcB0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32C_4x4); \\\n\
        VXC_DP4x4(tempB3, tmpA1, srcB1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16I16toF32D_4x4); \\\n\
        VXC_DP4x4(tmpZpScale, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16I16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempB0 + tmpZpScale.xxxx; \\\n\
        sum1 += tempA1 + tempB1 + tmpZpScale.yyyy; \\\n\
        sum2 += tempA2 + tempB2 + tmpZpScale.zzzz; \\\n\
        sum3 += tempA3 + tempB3 + tmpZpScale.wwww; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#endif\n\
GEMM_F16_QINT16_TO_QINT16(I16, vxc_short8)\n\
\n\
"; /* end of matrixmul_f16i16_i16_vx*/

static const char matrixmul_f16u8_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16U8MulZptoFp32_4x4;\n\
_viv_uniform float input1Scale;\n\
\n\
#define GEMM_F16_QINT_TO_F16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toF16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    half4 valC; \\\n\
    vxc_short8 srcA0, srcA1, outC; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    read_type srcB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempZp; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempZp, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16U8MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempZp.x; \\\n\
        sum1 += tempA1 + tempZp.y; \\\n\
        sum2 += tempA2 + tempZp.z; \\\n\
        sum3 += tempA3 + tempZp.w; \\\n\
    } \\\n\
    sum0 *= input1Scale; \\\n\
    sum1 *= input1Scale; \\\n\
    sum2 *= input1Scale; \\\n\
    sum3 *= input1Scale; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_F16_QINT_TO_F16(U8, vxc_uchar16)\n\
GEMM_F16_QINT_TO_F16(I8, vxc_char16)\n\
\n\
#define GEMM_F16_QINT16_TO_F16(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##toF16(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    half4 valC; \\\n\
    vxc_short8 srcA, outC; \\\n\
    vxc_half8 tmpA; \\\n\
    read_type srcB; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
    short in1_zp; \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA1, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA2, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA, srcA, 16); \\\n\
        VXC_DP4x4(tempA3, tmpA, tmpA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_F16_QINT16_TO_F16(I16, vxc_short8)\n\
"; /* end of matrixmul_f16u8_f16_vx*/

static const char matrixmul_f16u8_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32_4x4;\n\
_viv_uniform VXC_512Bits uniGemmF16U8toF32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemmFp16U8MulZptoFp32_4x4;\n\
_viv_uniform float in1outScale;\n\
\n\
#define GEMM_F16_QINT_TO_QINT(src1_type_name, read_type) \\\n\
__kernel void gemm_F16##src1_type_name##to##src1_type_name(image2d_array_t inputA, \\\n\
        image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_short8 srcA0, srcA1; \\\n\
    vxc_half8 tmpA0, tmpA1; \\\n\
    read_type srcB, outC; \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a); \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b); \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempZp; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpA0, srcA0, 16); \\\n\
        _viv_asm(COPY, tmpA1, srcA1, 16); \\\n\
        VXC_DP4x4(tempA0, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA1, tmpA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32_4x4); \\\n\
        VXC_DP4x4(tempA3, tmpA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGemmF16U8toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempZp, tmpA0, tmpA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16U8MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempZp.x; \\\n\
        sum1 += tempA1 + tempZp.y; \\\n\
        sum2 += tempA2 + tempZp.z; \\\n\
        sum3 += tempA3 + tempZp.w; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in1outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in1outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_F16_QINT_TO_QINT(U8, vxc_uchar16)\n\
GEMM_F16_QINT_TO_QINT(I8, vxc_char16)\n\
"; /* end of matrixmul_f16u8_u8_vx*/

static const char matrixmul_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
#define GEMM_QINT_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read_type srcA, srcB, outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_QINT_TO_QINT(I16, vxc_short8)\n\
"; /* end of matrixmul_i16_vx*/

static const char matrixmul_transA_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
#define GEMM_TRANSA_QINT(src0_type_name, src1_type_name, dst_type_name, read0_type, read1_type, write_type) \\\n\
__kernel void gemm_transa_##src0_type_name##src1_type_name##to##dst_type_name( \\\n\
            image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
            int transposeA, int transposeB, int adjointA, int adjointB, \\\n\
            uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read0_type srcA; \\\n\
    read1_type srcB; \\\n\
    write_type outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_float4 sum0 = (vxc_float4)(0); \\\n\
    vxc_float4 sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0); \\\n\
    vxc_float4 sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    vxc_float4 tempA0; \\\n\
    vxc_float4 tempB0; \\\n\
 \\\n\
    for(coord_a.y = 0, coord_b.y = 0; coord_a.y < K;) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.y++; \\\n\
        coord_b.y++; \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0); \\\n\
        sum1 = (sum1 + tempA0.y * tempB0); \\\n\
        sum2 = (sum2 + tempA0.z * tempB0); \\\n\
        sum3 = (sum3 + tempA0.w * tempB0); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_TRANSA_QINT(U8, U8, U8, vxc_uchar16, vxc_uchar16, vxc_uchar16)\n\
GEMM_TRANSA_QINT(I8, I8, I8, vxc_char16, vxc_char16, vxc_char16)\n\
GEMM_TRANSA_QINT(I16, I16, I16, vxc_short8, vxc_short8, vxc_short8)\n\
\n\
#define GEMM_TRANSA_INPUTB_F16(src0_type_name, read0_type) \\\n\
__kernel void gemm_transa_##src0_type_name##F16to##src0_type_name( \\\n\
                        image2d_array_t inputA, \\\n\
                        image2d_array_t inputB, \\\n\
                        image2d_array_t output, \\\n\
                                    int transposeA, \\\n\
                                    int transposeB, \\\n\
                                    int adjointA, \\\n\
                                    int adjointB, \\\n\
                        uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read0_type srcA, outC; \\\n\
    vxc_short8 srcB; \\\n\
    vxc_half8 tmpB; \\\n\
 \\\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
 \\\n\
    vxc_float4 sum0 = (vxc_float4)(0); \\\n\
    vxc_float4 sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0); \\\n\
    vxc_float4 sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    short in0_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
 \\\n\
    vxc_float4 tempA0; \\\n\
    vxc_float4 tempB0; \\\n\
 \\\n\
    for(coord_a.y = 0, coord_b.y = 0; coord_a.y < K;) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.y++; \\\n\
        coord_b.y++; \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB0,tmpB,tmpB,VXC_MODIFIER(0,3,0,VXC_RM_TowardZero,0),uniConvert1stFp16ToFp32_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0); \\\n\
        sum1 = (sum1 + tempA0.y * tempB0); \\\n\
        sum2 = (sum2 + tempA0.z * tempB0); \\\n\
        sum3 = (sum3 + tempA0.w * tempB0); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_TRANSA_INPUTB_F16(U8, vxc_uchar16)\n\
GEMM_TRANSA_INPUTB_F16(I8, vxc_char16)\n\
GEMM_TRANSA_INPUTB_F16(I16, vxc_short8)\n\
\n\
__kernel void gemm_transa_F16F16toF16(\n\
                        image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    uint gidy = get_global_id(1);\n\
\n\
    half4 valC;\n\
    vxc_short8 srcA, srcB, outC;\n\
    vxc_half8 tmpA, tmpB;\n\
\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    vxc_float4 tempA0;\n\
    vxc_float4 tempB0;\n\
\n\
    for(coord_a.y = 0, coord_b.y = 0; coord_a.y < K;)\n\
    {\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.y++;\n\
        coord_b.y++;\n\
        _viv_asm(COPY, tmpA, srcA, 16);\n\
        VXC_DP4x4(tempA0,tmpA,tmpA,VXC_MODIFIER(0,3,0,VXC_RM_TowardZero,0),uniConvert1stFp16ToFp32_4x4);\n\
        _viv_asm(COPY, tmpB, srcB, 16);\n\
        VXC_DP4x4(tempB0,tmpB,tmpB,VXC_MODIFIER(0,3,0,VXC_RM_TowardZero,0),uniConvert1stFp16ToFp32_4x4);\n\
\n\
        sum0 = (sum0 + tempA0.x * tempB0);\n\
        sum1 = (sum1 + tempA0.y * tempB0);\n\
        sum2 = (sum2 + tempA0.z * tempB0);\n\
        sum3 = (sum3 + tempA0.w * tempB0);\n\
    }\n\
    coord_b.y = gidy;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr);\n\
    _viv_asm(CONV, valC, sum0);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum1);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum2);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_b.y++;\n\
    _viv_asm(CONV, valC, sum3);\n\
    _viv_asm(COPY, outC, valC, 16);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}"; /* end of matrixmul_transA_vx*/

static const char matrixmul_transB_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************gemm transposeB fp16 fp16 to fp16*************************/\n\
_viv_uniform VXC_512Bits uniFp16MulFp16AddtoFp32_dp8x2;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
__kernel void gemm_transb_F16F16toF16(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_short8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_short8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        _viv_asm(COPY, halfB0, srcB0, 16);\n\
        _viv_asm(COPY, halfB1, srcB1, 16);\n\
        _viv_asm(COPY, halfB2, srcB2, 16);\n\
        _viv_asm(COPY, halfB3, srcB3, 16);\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        _viv_asm(COPY, halfA0, srcA0, 16);\n\
        _viv_asm(COPY, halfA1, srcA1, 16);\n\
        _viv_asm(COPY, halfA2, srcA2, 16);\n\
        _viv_asm(COPY, halfA3, srcA3, 16);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    half4 halfDst;\n\
    vxc_short8 valDst;\n\
    _viv_asm(CONV, halfDst, sum0);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    _viv_asm(CONV, halfDst, sum1);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    _viv_asm(CONV, halfDst, sum2);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    _viv_asm(CONV, halfDst, sum3);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of matrixmul_transB_f16_vx*/

static const char matrixmul_transB_f16_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************gemm transposeB fp16 uint8 to fp16*************************/\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform VXC_512Bits uniU8SubZptoFp16_dp2x8;\n\
_viv_uniform VXC_512Bits uniFp16MulFp16AddtoFp32_dp8x2;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
__kernel void gemm_transb_F16U8toF16(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_short8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        _viv_asm(COPY, halfA0, srcA0, 16);\n\
        _viv_asm(COPY, halfA1, srcA1, 16);\n\
        _viv_asm(COPY, halfA2, srcA2, 16);\n\
        _viv_asm(COPY, halfA3, srcA3, 16);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    half4 halfDst;\n\
    vxc_short8 valDst;\n\
    sum0 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum0);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum1 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum1);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum2 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum2);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum3 *= input1Scale;\n\
    _viv_asm(CONV, halfDst, sum3);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
/***********************gemm transposeB fp16 uint8 to uint8***********************************/\n\
_viv_uniform float scaleIn2divOut;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float output_ZP;\n\
\n\
__kernel void gemm_transb_F16U8toU8(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_short8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        _viv_asm(COPY, halfA0, srcA0, 16);\n\
        _viv_asm(COPY, halfA1, srcA1, 16);\n\
        _viv_asm(COPY, halfA2, srcA2, 16);\n\
        _viv_asm(COPY, halfA3, srcA3, 16);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    vxc_uchar8 valDst;\n\
    tmpOut0 = convert_int4_rte(sum0 * scaleIn2divOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum1 * scaleIn2divOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    tmpOut0 = convert_int4_rte(sum2 * scaleIn2divOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum3 * scaleIn2divOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of matrixmul_transB_f16_mix_vx*/

static const char matrixmul_transB_u8_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
/********************gemm transposeB uint8 uint8 to fp16*************************/\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform float inScaleMul;\n\
_viv_uniform VXC_512Bits uniU8SubZptoFp16_dp2x8;\n\
_viv_uniform VXC_512Bits uniFp16MulFp16AddtoFp32_dp8x2;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
__kernel void gemm_transb_U8U8toF16(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4);\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_uchar8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        VXC_DP2x8(halfA0, srcA0, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA1, srcA1, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA2, srcA2, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA3, srcA3, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    half4 halfDst;\n\
    vxc_short8 valDst;\n\
    sum0 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum0);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum1 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum1);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum2 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum2);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    sum3 *= inScaleMul;\n\
    _viv_asm(CONV, halfDst, sum3);\n\
    _viv_asm(COPY, valDst, halfDst, 16);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
/********************gemm transposeB uint8 uint8 to uint8*************************/\n\
_viv_uniform float inScaledivOut;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float output_ZP;\n\
\n\
__kernel void gemm_transb_U8U8toU8(image2d_array_t inputA,\n\
                        image2d_array_t inputB,\n\
                        image2d_array_t output,\n\
                                    int transposeA,\n\
                                    int transposeB,\n\
                                    int adjointA,\n\
                                    int adjointB,\n\
                        uint M, uint K, uint N)\n\
{\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_a = (int4)(0, coord_out.y, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, coord_out.x, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    vxc_float4 sum0 = (vxc_float4)(0);\n\
    vxc_float4 sum1 = (vxc_float4)(0);\n\
    vxc_float4 sum2 = (vxc_float4)(0);\n\
    vxc_float4 sum3 = (vxc_float4)(0);\n\
\n\
    int8 inputA_desc, inputB_desc, output_desc;\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc));\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0;\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc));\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0;\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);\n\
\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4);\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4);\n\
    for(coord_a.x = 0, coord_b.x = 0; coord_a.x < K;)\n\
    {\n\
        vxc_uchar8 srcA0,srcA1,srcA2,srcA3;\n\
        vxc_uchar8 srcB0,srcB1,srcB2,srcB3;\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB0, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB1, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB2, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, srcB3, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_a.x += 8;\n\
        coord_b.x += 8;\n\
\n\
        vxc_half8 halfA0,halfA1,halfA2,halfA3;\n\
        VXC_DP2x8(halfA0, srcA0, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA1, srcA1, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA2, srcA2, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfA3, srcA3, in0_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_half8 halfB0,halfB1,halfB2,halfB3;\n\
        VXC_DP2x8(halfB0, srcB0, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB1, srcB1, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB2, srcB2, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        VXC_DP2x8(halfB3, srcB3, in1_zp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
            uniU8SubZptoFp16_dp2x8);\n\
        vxc_float4 fpVal;\n\
        VXC_DP8x2(fpVal, halfA0, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA0, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum0 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA1, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA1, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum1 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA2, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA2, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum2 += fpVal;\n\
        VXC_DP8x2(fpVal, halfA3, halfB0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB1, VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB2, VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        VXC_DP8x2(fpVal, halfA3, halfB3, VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0),\n\
            uniFp16MulFp16AddtoFp32_dp8x2);\n\
        sum3 += fpVal;\n\
    }\n\
    vxc_int4 tmpOut0, tmpOut1;\n\
    vxc_uchar8 valDst;\n\
    tmpOut0 = convert_int4_rte(sum0 * inScaledivOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum1 * inScaledivOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    tmpOut0 = convert_int4_rte(sum2 * inScaledivOut + output_ZP);\n\
    tmpOut1 = convert_int4_rte(sum3 * inScaledivOut + output_ZP);\n\
    VXC_DP2x8(valDst, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_WriteImage2DArray(output, coord_out, valDst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of matrixmul_transB_u8_mix_vx*/

static const char matrixmul_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float mulKIn0In1Zp;\n\
_viv_uniform float inOutScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8U8toFp32Block4_4x4;\n\
_viv_uniform VXC_512Bits uniGemmU8U8MulZptoFp32_8x4;\n\
\n\
#define GEMM_QINT_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    read_type srcA0, srcA1, srcA2, srcA3, srcB, outC; \\\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp, mulKIn0In1Zp), sum1 = sum0; \\\n\
    vxc_float4 sum2 = sum0, sum3 = sum0; \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP8x4(tempB0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        sum0 += tempA0 + tempB0; \\\n\
        sum1 += tempA1 + tempB1; \\\n\
        sum2 += tempA2 + tempB2; \\\n\
        sum3 += tempA3 + tempB3; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = get_global_id(1); \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * inOutScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * inOutScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * inOutScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * inOutScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
GEMM_QINT_TO_QINT(U8, vxc_uchar16)\n\
GEMM_QINT_TO_QINT(I8, vxc_char16)\n\
"; /* end of matrixmul_u8_vx*/

static const char matrixmul_u8f16_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float input0Scale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Hi_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmFp16MulZptoFp32_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniGemm1stU8F16toF32Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemm2ndU8F16toF32Lo_4x4;\n\
_viv_uniform VXC_512Bits uniGemm1stU8F16toF32Hi_4x4;\n\
_viv_uniform VXC_512Bits uniGemm2ndU8F16toF32Hi_4x4;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_QINT_F16_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    half4 valC; \\\n\
    read_type srcA0, srcA1; \\\n\
    vxc_short8 outC; \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : gidz), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : gidz), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4(tmpZpScale, tmpB.hi, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tmpZpScale; \\\n\
        sum1 += tempA1 + tmpZpScale; \\\n\
        sum2 += tempA2 + tmpZpScale; \\\n\
        sum3 += tempA3 + tmpZpScale; \\\n\
    } \\\n\
    sum0 *= input0Scale; \\\n\
    sum1 *= input0Scale; \\\n\
    sum2 *= input0Scale; \\\n\
    sum3 *= input0Scale; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)gidz * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr);  \\\n\
 \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT_F16_TO_F16(U8, vxc_uchar16)\n\
GEMM_QINT_F16_TO_F16(I8, vxc_char16)\n\
GEMM_QINT_F16_TO_F16(I16, vxc_short8)\n\
#else\n\
#define GEMM_QINT_F16_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    half4 valC; \\\n\
    read_type srcA0, srcA1; \\\n\
    vxc_short8 outC; \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : gidz), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : gidz), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4(tempA0, srcA0, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempB0, srcA0, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempA1, srcA0, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempB1, srcA0, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempA2, srcA1, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempB2, srcA1, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Lo_4x4); \\\n\
        VXC_DP4x4(tempA3, srcA1, tmpB.hi, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm1stU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tempB3, srcA1, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemm2ndU8F16toF32Hi_4x4); \\\n\
        VXC_DP4x4(tmpZpScale, tmpB.hi, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tempB0 + tmpZpScale; \\\n\
        sum1 += tempA1 + tempB1 + tmpZpScale; \\\n\
        sum2 += tempA2 + tempB2 + tmpZpScale; \\\n\
        sum3 += tempA3 + tempB3 + tmpZpScale; \\\n\
    } \\\n\
    sum0 *= input0Scale; \\\n\
    sum1 *= input0Scale; \\\n\
    sum2 *= input0Scale; \\\n\
    sum3 *= input0Scale; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)gidz * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr);  \\\n\
 \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT_F16_TO_F16(U8, vxc_uchar16)\n\
GEMM_QINT_F16_TO_F16(I8, vxc_char16)\n\
GEMM_QINT_F16_TO_F16(I16, vxc_short8)\n\
#endif\n\
"; /* end of matrixmul_u8f16_f16_vx*/

static const char matrixmul_u8f16_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Lo_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmU8F16toF32Hi_4x4b;\n\
_viv_uniform VXC_512Bits uniGemmFp16MulZptoFp32_4x4;\n\
_viv_uniform float in0outScale;\n\
\n\
#if (VX_VERSION==2)\n\
#define GEMM_QINT_F16_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    uint gidz = get_global_id(2); \\\n\
    vxc_short16 srcB; \\\n\
    vxc_half16 tmpB; \\\n\
    read_type srcA0, srcA1, outC; \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : gidz), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : gidz), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3, tmpZpScale; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.hi, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB.lo, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        _viv_asm(COPY, tmpB.hi, srcB.hi, 16); \\\n\
        _viv_asm(COPY, tmpB.lo, srcB.lo, 16); \\\n\
        VXC_DP4x4_b(tempA0, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA1, tmpB.hi, tmpB.lo, srcA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4_b(tempA2, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Lo_4x4b); \\\n\
        VXC_DP4x4_b(tempA3, tmpB.hi, tmpB.lo, srcA1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8F16toF32Hi_4x4b); \\\n\
        VXC_DP4x4(tmpZpScale, tmpB.hi, tmpB.lo, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmFp16MulZptoFp32_4x4); \\\n\
        sum0 += tempA0 + tmpZpScale; \\\n\
        sum1 += tempA1 + tmpZpScale; \\\n\
        sum2 += tempA2 + tmpZpScale; \\\n\
        sum3 += tempA3 + tmpZpScale; \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * in0outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * in0outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * in0outScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * in0outScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#else\n\
#define GEMM_QINT_F16_TO_QINT(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##F16to##src0_type_name( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    vxc_short8 srcB; \\\n\
    vxc_half8 tmpB; \\\n\
    half4 valB; \\\n\
    read_type srcA, outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0), sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0), sum3 = (vxc_float4)(0); \\\n\
    short in0_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB0, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB1, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB2, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        _viv_asm(COPY, tmpB, srcB, 16); \\\n\
        VXC_DP4x4(tempB3, tmpB, tmpB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvert1stFp16ToFp32_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    vxc_int4 tmpOut0, tmpOut1; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    tmpOut0 = convert_int4_rte(sum0 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum1 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    tmpOut0 = convert_int4_rte(sum2 * outputScale + output_ZP); \\\n\
    tmpOut1 = convert_int4_rte(sum3 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(outC, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0123, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord_b.y++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s4567, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
#endif\n\
GEMM_QINT_F16_TO_QINT(U8, vxc_uchar16)\n\
GEMM_QINT_F16_TO_QINT(I8, vxc_char16)\n\
GEMM_QINT_F16_TO_QINT(I16, vxc_short8)\n\
\n\
"; /* end of matrixmul_u8f16_u8_vx*/

static const char matrixmul_u8u8_f16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input0_ZP;\n\
_viv_uniform int input1_ZP;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32B_4x4;\n\
_viv_uniform int ac2zero;\n\
_viv_uniform int bc2zero;\n\
\n\
_viv_uniform VXC_512Bits uniGemmU8U8toFp32Block4_4x4;\n\
_viv_uniform VXC_512Bits uniGemmU8U8MulZptoFp32_8x4;\n\
_viv_uniform float input01Scale;\n\
\n\
#define GEMM_QINT_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read_type srcA0, srcA1, srcA2, srcA3, srcB; \\\n\
    half4 valC; \\\n\
    vxc_short8 outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0); \\\n\
    vxc_float4 sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0); \\\n\
    vxc_float4 sum3 = (vxc_float4)(0); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA0, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA1, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA2, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(8, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcA3, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(12, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP4x4(tempA3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8toFp32Block4_4x4); \\\n\
        VXC_DP8x4(tempB0, srcA0, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB1, srcA1, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB2, srcA2, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        VXC_DP8x4(tempB3, srcA3, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniGemmU8U8MulZptoFp32_8x4); \\\n\
        sum0 += tempA0 + tempB0; \\\n\
        sum1 += tempA1 + tempB1; \\\n\
        sum2 += tempA2 + tempB2; \\\n\
        sum3 += tempA3 + tempB3; \\\n\
    } \\\n\
    sum0 *= input01Scale; \\\n\
    sum1 *= input01Scale; \\\n\
    sum2 *= input01Scale; \\\n\
    sum3 *= input01Scale; \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT_TO_F16(U8, vxc_uchar16)\n\
GEMM_QINT_TO_F16(I8, vxc_char16)\n\
\n\
#define GEMM_QINT16_TO_F16(src0_type_name, read_type) \\\n\
__kernel void gemm_##src0_type_name##src0_type_name##toF16( \\\n\
        image2d_array_t inputA, image2d_array_t inputB, image2d_array_t output, \\\n\
        int transposeA, int transposeB, int adjointA, int adjointB, uint M, uint K, uint N) \\\n\
{ \\\n\
    uint gidy = get_global_id(1); \\\n\
    read_type srcA, srcB; \\\n\
    half4 valC; \\\n\
    vxc_short8 outC; \\\n\
 \\\n\
    int4 coord_a = (int4)(0, gidy, (ac2zero ? 0 : get_global_id(2)), 0); \\\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0); \\\n\
    vxc_float4 sum0 = (vxc_float4)(0); \\\n\
    vxc_float4 sum1 = (vxc_float4)(0); \\\n\
    vxc_float4 sum2 = (vxc_float4)(0); \\\n\
    vxc_float4 sum3 = (vxc_float4)(0); \\\n\
    short in0_zp, in1_zp; \\\n\
    _viv_asm(COPY, in0_zp, input0_ZP, 4); \\\n\
    _viv_asm(COPY, in1_zp, input1_ZP, 4); \\\n\
 \\\n\
    int8 inputA_desc, inputB_desc, output_desc; \\\n\
    _viv_asm(COPY, inputA_desc, inputA, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)coord_a.z * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord_a.w, baseAddr_a);  \\\n\
    _viv_asm(COPY, inputB_desc, inputB, sizeof(inputB_desc)); \\\n\
    int baseAddr_b = (int)coord_b.z * inputB_desc.s4 + inputB_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr_b);  \\\n\
 \\\n\
    for(coord_a.x = 0, coord_b.y = 0; coord_a.x < K;) \\\n\
    { \\\n\
        vxc_float4 tempA0, tempA1, tempA2, tempA3; \\\n\
        vxc_float4 tempB0, tempB1, tempB2, tempB3; \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA0, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB0, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 1), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA1, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB1, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 2), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tempA2, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB2, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
 \\\n\
        VXC_OP4(img_load_3d, srcA, inputA, coord_a.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_OP4(img_load_3d, srcB, inputB, coord_b.xywz, VXC_5BITOFFSET_XY(0, 3), \\\n\
                    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord_a.x += 4; \\\n\
        coord_b.y += 4; \\\n\
        VXC_DP4x4(tempA3, srcA, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32_4x4); \\\n\
        VXC_DP4x4(tempB3, srcB, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                    uniConvertUint8SubZpToFp32B_4x4); \\\n\
        sum0 = (sum0 + tempA0.x * tempB0 + tempA0.y * tempB1 + tempA0.z * tempB2 + tempA0.w * tempB3); \\\n\
        sum1 = (sum1 + tempA1.x * tempB0 + tempA1.y * tempB1 + tempA1.z * tempB2 + tempA1.w * tempB3); \\\n\
        sum2 = (sum2 + tempA2.x * tempB0 + tempA2.y * tempB1 + tempA2.z * tempB2 + tempA2.w * tempB3); \\\n\
        sum3 = (sum3 + tempA3.x * tempB0 + tempA3.y * tempB1 + tempA3.z * tempB2 + tempA3.w * tempB3); \\\n\
    } \\\n\
    coord_b.y = gidy; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_b.w, baseAddr); \\\n\
    _viv_asm(CONV, valC, sum0); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum1); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum2); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
    coord_b.y++; \\\n\
    _viv_asm(CONV, valC, sum3); \\\n\
    _viv_asm(COPY, outC, valC, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_b.xywz, outC.s0246, \\\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));  \\\n\
}\n\
GEMM_QINT16_TO_F16(I16, vxc_short8)\n\
"; /* end of matrixmul_u8u8_f16_vx*/

static const char maximum_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void maximum_F16F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_F16F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    coord.z ++;\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
__kernel void maximum_F16F16toI8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1;\n\
    vxc_char8  dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_F16F16toI8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1;\n\
    vxc_char8  dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part1_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_1_part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_1_part1_2x8;\n\
__kernel void maximum_I8I8toI8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char16 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part1_2x8);\n\
    dst = max(src0, src1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I8I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_char16 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.z ++;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part1_2x8);\n\
    dst = max(src0, src1);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
__kernel void maximum_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Lo_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Hi_2x8);\n\
    dst = max(src0, src1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Lo_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Hi_2x8);\n\
    dst = max(src0, src1);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI16toI16_0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI16toI16_1_2x8;\n\
__kernel void maximum_I16I16toI16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_1_2x8);\n\
    dst = max(src0, src1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I16I16toI16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.z ++;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_1_2x8);\n\
    dst = max(src0, src1);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of maximum_vx*/

static const char maximum_fp16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part1_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
__kernel void maximum_I8F16toI8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char16 src0, src2, dst;\n\
    vxc_short8 src1, src3, src4, src5;\n\
    vxc_half8 data0, data1, data2, data3;\n\
    vxc_char16 tmp0, tmp1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src4, input1, coord, VXC_5BITOFFSET_XY(8, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
    _viv_asm(COPY, data1, src4, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    VXC_DP2x8(tmp0, data1, data1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    dst = max(src0, tmp0);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I8F16toI8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_char16 src0, src2, dst;\n\
    vxc_short8 src1, src3, src4, src5;\n\
    vxc_half8 data0, data1, data2, data3;\n\
    vxc_char16 tmp0;\n\
\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src4, input1, coord.xy, VXC_5BITOFFSET_XY(8, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
    _viv_asm(COPY, data1, src4, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    VXC_DP2x8(tmp0, data1, data1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    dst = max(src0, tmp0);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I8F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt8toFp16_2x8);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I8F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_char8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
    VXC_ReadImage(vec0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt8toFp16_2x8);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
__kernel void maximum_U8F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_U8F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
__kernel void maximum_U8F16toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar16 src0, dst0, dst1;\n\
    vxc_ushort8 src1, src2;\n\
    vxc_half8 data1, data2;\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src2, input1, coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    _viv_asm(COPY, data2, src2, 16);\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data2, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = max(dst0, dst1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_U8F16toU8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, dst0, dst1;\n\
    vxc_ushort8 src1, src2;\n\
    vxc_half8 data1, data2;\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input1, coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    _viv_asm(COPY, data2, src2, 16);\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data2, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = max(dst0, dst1);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_F16F16toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src0, src1;\n\
    vxc_half8 data0, data1;\n\
    vxc_uchar16 dst0, dst1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    _viv_asm(COPY, data1, src1, 16);\n\
\n\
    vxc_ushort8 mp1;\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, data0, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = max(dst0, dst1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_F16F16toU8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_ushort8 src0, src1;\n\
    vxc_half8 data0, data1;\n\
    vxc_uchar16 dst0, dst1;\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    _viv_asm(COPY, data1, src1, 16);\n\
\n\
    vxc_ushort8 mp1;\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, data0, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = max(dst0, dst1);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of maximum_fp16_vx*/

static const char maximum_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt16toFp16_2x8;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndFp16ToFp32_4x4;\n\
\n\
\n\
__kernel void maximum_I16F16toI16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, tmp0, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt16_2x8);\n\
    dst = max(src0, tmp0);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I16F16toI16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, tmp0, dst;\n\
    vxc_half8 data0;\n\
\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt16_2x8);\n\
    dst = max(src0, tmp0);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I16F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt16toFp16_2x8);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_I16F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt16toFp16_2x8);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_F16F16toI16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1;\n\
    vxc_short8 dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    int4 tmpDst0, tmpDst1;\n\
    float4 tmpData0, tmpData1;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert1stFp16ToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert2ndFp16ToFp32_4x4);\n\
    tmpDst0 = convert_int4_rte(tmpData0 * outputScale + output_zp);\n\
    tmpDst1 = convert_int4_rte(tmpData1 * outputScale + output_zp);\n\
    VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void maximum_F16F16toI16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1;\n\
    vxc_short8 dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMax3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    int4 tmpDst0, tmpDst1;\n\
    float4 tmpData0, tmpData1;\n\
    VXC_DP4x4(tmpData0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert1stFp16ToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert2ndFp16ToFp32_4x4);\n\
    tmpDst0 = convert_int4_rte(tmpData0 * outputScale + output_zp);\n\
    tmpDst1 = convert_int4_rte(tmpData1 * outputScale + output_zp);\n\
    VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}"; /* end of maximum_i16_vx*/

static const char minimum_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void minimum_F16F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_F16F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    coord.z ++;\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
\n\
__kernel void minimum_F16F16toI8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1;\n\
    vxc_char8  dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_F16F16toI8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1;\n\
    vxc_char8  dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src0, vec0, 16);\n\
    VXC_ReadImage(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    VXC_DP2x8(dst, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part1_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_1_part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_1_part1_2x8;\n\
__kernel void minimum_I8I8toI8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char16 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part1_2x8);\n\
    dst = min(src0, src1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I8I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_char16 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.z ++;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_1_part1_2x8);\n\
    dst = min(src0, src1);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift1_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
__kernel void minimum_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Lo_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Hi_2x8);\n\
    dst = min(src0, src1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(src0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Lo_2x8);\n\
    VXC_DP2x8(src1, src1, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift1_Hi_2x8);\n\
    dst = min(src0, src1);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI16toI16_0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI16toI16_1_2x8;\n\
__kernel void minimum_I16I16toI16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_1_2x8);\n\
    dst = min(src0, src1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I16I16toI16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.z ++;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_0_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_1_2x8);\n\
    dst = min(src0, src1);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of minimum_vx*/

static const char minimum_fp16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_0_part1_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt8toFp16_2x8;\n\
\n\
__kernel void minimum_I8F16toI8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char16 src0, src2, dst;\n\
    vxc_short8 src1, src3, src4, src5;\n\
    vxc_half8 data0, data1, data2, data3;\n\
    vxc_char16 tmp0, tmp1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src4, input1, coord, VXC_5BITOFFSET_XY(8, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
    _viv_asm(COPY, data1, src4, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    VXC_DP2x8(tmp0, data1, data1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    dst = min(src0, tmp0);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I8F16toI8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_char16 src0, src2, dst;\n\
    vxc_short8 src1, src3, src4, src5;\n\
    vxc_half8 data0, data1, data2, data3;\n\
    vxc_char16 tmp0;\n\
\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src4, input1, coord.xy, VXC_5BITOFFSET_XY(8, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
    _viv_asm(COPY, data1, src4, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part0_2x8);\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_0_part1_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    VXC_DP2x8(tmp0, data1, data1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt8_2x8);\n\
    dst = min(src0, tmp0);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I8F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt8toFp16_2x8);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I8F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_char8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
    VXC_ReadImage(vec0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt8toFp16_2x8);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform int2 multAndoutZP0;//[0:15] multiplier, [31:63] output zp\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
\n\
__kernel void minimum_U8F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_U8F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar8 vec0, vec2;\n\
    vxc_short8 vec1, vec3, dst;\n\
    vxc_half8  src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    vxc_ushort8 ms0;\n\
    _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
    VXC_DP2x8(src0, vec0, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
                uniU8MulAndPostShift_0_Lo_2x8);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift0_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniConvertFp16toU8_2x8;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
__kernel void minimum_U8F16toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar16 src0, dst0, dst1;\n\
    vxc_ushort8 src1, src2;\n\
    vxc_half8 data1, data2;\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src2, input1, coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    _viv_asm(COPY, data2, src2, 16);\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data2, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = min(dst0, dst1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_U8F16toU8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, dst0, dst1;\n\
    vxc_ushort8 src1, src2;\n\
    vxc_half8 data1, data2;\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input1, coord, VXC_5BITOFFSET_XY(8, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    _viv_asm(COPY, data2, src2, 16);\n\
\n\
    vxc_ushort8 mp0, mp1;\n\
    _viv_asm(COPY, mp0, multAndoutZP0, 16);\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Lo_2x8);\n\
    VXC_DP2x8(dst0, src0, mp0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8MulAndPostShift0_Hi_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data2, mp1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = min(dst0, dst1);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_F16F16toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src0, src1;\n\
    vxc_half8 data0, data1;\n\
    vxc_uchar16 dst0, dst1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    _viv_asm(COPY, data1, src1, 16);\n\
\n\
    vxc_ushort8 mp1;\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, data0, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = min(dst0, dst1);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_F16F16toU8_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_ushort8 src0, src1;\n\
    vxc_half8 data0, data1;\n\
    vxc_uchar16 dst0, dst1;\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    _viv_asm(COPY, data1, src1, 16);\n\
\n\
    vxc_ushort8 mp1;\n\
    _viv_asm(COPY, mp1, multAndoutZP1, 16);\n\
    VXC_DP2x8(dst0, data0, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    VXC_DP2x8(dst1, data1, mp1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertFp16toU8_2x8);\n\
    dst0 = min(dst0, dst1);\n\
\n\
    VXC_WriteImage(output, coord, dst0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of minimum_fp16_vx*/

static const char minimum_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertI16toI16_2x8;\n\
_viv_uniform VXC_512Bits uinConvertFp16ToInt16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertInt16toFp16_2x8;\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform float output_zp;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stFp16ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvert2ndFp16ToFp32_4x4;\n\
\n\
__kernel void minimum_I16F16toI16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, tmp0, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt16_2x8);\n\
    dst = min(src0, tmp0);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I16F16toI16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, tmp0, dst;\n\
    vxc_half8 data0;\n\
\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src1, 16);\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI16toI16_2x8);\n\
    VXC_DP2x8(tmp0, data0, data0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uinConvertFp16ToInt16_2x8);\n\
    dst = min(src0, tmp0);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I16F16toF16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
\n\
    VXC_ReadImage2DArray(vec0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(vec1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt16toFp16_2x8);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_I16F16toF16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 vec0, vec1, dst;\n\
    vxc_half8  src0, src1;\n\
    VXC_ReadImage(vec0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(vec1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, src1, vec1, 16);\n\
\n\
    VXC_DP2x8(src0, vec0, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt16toFp16_2x8);\n\
\n\
    VXC_VertMin3_Half(src0, src0, src1, src1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
    _viv_asm(COPY, dst, src0, 16);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_F16F16toI16\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    _viv_asm(COPY, data1, src1, 16);\n\
\n\
    VXC_VertMin3_Half(data0, data0, data1, data1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
\n\
    int4 tmpDst0, tmpDst1;\n\
    float4 tmpData0, tmpData1;\n\
    VXC_DP4x4(tmpData0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert1stFp16ToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert2ndFp16ToFp32_4x4);\n\
    tmpDst0 = convert_int4_rte(tmpData0 * outputScale + output_zp);\n\
    tmpDst1 = convert_int4_rte(tmpData1 * outputScale + output_zp);\n\
    VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void minimum_F16F16toI16_2D\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0, data1;\n\
\n\
    VXC_ReadImage(src0, input0, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    _viv_asm(COPY, data1, src1, 16);\n\
\n\
    VXC_VertMin3_Half(data0, data0, data1, data1, VXC_MODIFIER_CLAMP(0, 7, 0, 0));\n\
\n\
    int4 tmpDst0, tmpDst1;\n\
    float4 tmpData0, tmpData1;\n\
    VXC_DP4x4(tmpData0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert1stFp16ToFp32_4x4);\n\
    VXC_DP4x4(tmpData1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 0), uniConvert2ndFp16ToFp32_4x4);\n\
    tmpDst0 = convert_int4_rte(tmpData0 * outputScale + output_zp);\n\
    tmpDst1 = convert_int4_rte(tmpData1 * outputScale + output_zp);\n\
    VXC_DP2x8(dst, tmpDst0, tmpDst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
}"; /* end of minimum_i16_vx*/

static const char moments_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform float zpScaleSqr_i16;\n\
_viv_uniform float zpScale2_i16;\n\
_viv_uniform float sumScale_i16;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
#define MOMENTS_AXIS0_QINT(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toF16( \\\n\
    image2d_array_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(0, gidy, gidz, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
 \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidy, gidz); \\\n\
    half4 tmpData; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpData, mean_vari0); \\\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
 \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
MOMENTS_AXIS0_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS0_QINT(I8, vxc_char16)\n\
\n\
#define MOMENTS_AXIS0_QINT_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toF16_2D( \\\n\
    image2d_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int2 coord = (int2)(0, gidy); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
 \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidy, 0); \\\n\
    half4 tmpData; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpData, mean_vari0); \\\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
 \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS0_QINT_2D(U8, vxc_uchar16)\n\
MOMENTS_AXIS0_QINT_2D(I8, vxc_char16)\n\
\n\
__kernel void moments_axis0_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(0, gidy, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 sumsqr0;\n\
    vxc_float4 mean_vari0 = (vxc_float4)(0);\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP8x2(sumsqr0, in_h0, in_h0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        mean_vari0 += sumsqr0;\n\
    }\n\
\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_F16toF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int2 coord = (int2)(0, gidy);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 sumsqr0;\n\
    vxc_float4 mean_vari0 = (vxc_float4)(0);\n\
\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP8x2(sumsqr0, in_h0, in_h0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        mean_vari0 += sumsqr0;\n\
    }\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, 0);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(0, gidy, gidz, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr0;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0;\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP8x2(sumsqr0, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        tmpSum += sumsqr0.x;\n\
        sqr += (sumsqr0.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr0.x);\n\
    }\n\
    sum = tmpSum * input_scale + sumScale_i16;\n\
\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0);\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis0_I16toF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidy = get_global_id(0);\n\
    int2 coord = (int2)(0, gidy);\n\
    vxc_short8 src0;\n\
    float4 sumsqr0;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0;\n\
    for(coord.x = 0; coord.x < width; coord.x += 8)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP8x2(sumsqr0, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                uniFp16SumSqr_dp8x2);\n\
        tmpSum += sumsqr0.x;\n\
        sqr += (sumsqr0.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr0.x);\n\
    }\n\
    sum = tmpSum * input_scale + sumScale_i16;\n\
\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0);\n\
    mean_vari0 *= dimRatio;\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0;\n\
\n\
    int2 coord_out = (int2)(gidy, 0);\n\
\n\
    half4 tmpData;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpData, mean_vari0);\n\
    VXC_DP2x8(tmpVal, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of moments_axis0_vx*/

static const char moments_axis01_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform float zpScaleSqr_i16;\n\
_viv_uniform float zpScale2_i16;\n\
_viv_uniform float sumScale_i16;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
#define MOMENTS_AXIS01_QINT(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toF16( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(gidz, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        half4 tmpMean, tmpVari; \\\n\
        vxc_half8 tmpVal; \\\n\
        vxc_short8 dst; \\\n\
        _viv_asm(CONV, tmpMean, mean); \\\n\
        _viv_asm(CONV, tmpVari, vari); \\\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari, \\\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
        _viv_asm(COPY, dst, tmpVal, 16); \\\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS01_QINT(I8, vxc_char16)\n\
\n\
#define MOMENTS_AXIS01_QINT_2D(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toF16_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        half4 tmpMean, tmpVari; \\\n\
        vxc_half8 tmpVal; \\\n\
        vxc_short8 dst; \\\n\
        _viv_asm(CONV, tmpMean, mean); \\\n\
        _viv_asm(CONV, tmpVari, vari); \\\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari, \\\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
        _viv_asm(COPY, dst, tmpVal, 16); \\\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT_2D(U8, vxc_uchar16)\n\
MOMENTS_AXIS01_QINT_2D(I8, vxc_char16)\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_F16toF16(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0.0f;\n\
        float sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_F16toF16_2D(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            _viv_asm(COPY, in_h, src0, 16);\n\
            VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSumSqr += sumsqr;\n\
        }\n\
    }\n\
\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = 0.0f;\n\
        float sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_I16toF16(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        float tmpSum = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSum += sumsqr.x;\n\
            sqr += (sumsqr.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr.x);\n\
        }\n\
        sum += tmpSum * input_scale + sumScale_i16;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0.0f;\n\
        sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_I16toF16_2D(\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari,\n\
    int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
    {\n\
        float tmpSum = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            coord.y++;\n\
            VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
            tmpSum += sumsqr.x;\n\
            sqr += (sumsqr.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr.x);\n\
        }\n\
        sum += tmpSum * input_scale + sumScale_i16;\n\
    }\n\
\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0.0f;\n\
        sqr = 0.0f;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
        float4 mean_vari;\n\
        mean_vari.x = sum * dimRatio;\n\
        mean_vari.y = sqr * dimRatio;\n\
        mean_vari.y = mean_vari.y - mean_vari.x * mean_vari.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean_vari);\n\
        //_viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpMean,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
"; /* end of moments_axis01_vx*/

static const char moments_axis012_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
_viv_uniform float zpScaleSqr_i16;\n\
_viv_uniform float zpScale2_i16;\n\
_viv_uniform float sumScale_i16;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform VXC_512Bits uniFp16SumSqr_dp8x2;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
#define MOMENTS_AXIS012_QINT(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_##src0_type_name##toF16( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int4 coord = (int4)(gidx, 0, 0, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
        { \\\n\
            int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
            for(coord.y = 0; coord.y < height;) \\\n\
            { \\\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
                coord.y++; \\\n\
                VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
                tmpSum += (tmpSum1); \\\n\
                VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
                tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
            } \\\n\
            sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
            sum += (tmpSum + sumInZp) * input_scale; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        half4 tmpMean, tmpVari; \\\n\
        vxc_half8 tmpVal; \\\n\
        vxc_short8 dst; \\\n\
        _viv_asm(CONV, tmpMean, mean); \\\n\
        _viv_asm(CONV, tmpVari, vari); \\\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari, \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
        _viv_asm(COPY, dst, tmpVal, 16); \\\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
\n\
MOMENTS_AXIS012_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS012_QINT(I8, vxc_char16)\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    vxc_float4 sumsqr;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
        {\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
                coord.y++;\n\
                _viv_asm(COPY, in_h, src0, 16);\n\
                VXC_DP8x2(sumsqr, in_h, in_h, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
                tmpSumSqr += sumsqr;\n\
            }\n\
        }\n\
    }\n\
    lcl_sum[lidx] = tmpSumSqr.x;\n\
    lcl_sqr[lidx] = tmpSumSqr.y;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        float sum = (float)(0);\n\
        float sqr = (float)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_I16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0) << 3;\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    vxc_short8 src0;\n\
    float4 sumsqr;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
    vxc_float4 tmpSumSqr = (vxc_float4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 128)\n\
        {\n\
            float tmpSum = 0;\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
                coord.y++;\n\
                VXC_DP8x2(sumsqr, src0, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0),\\\n\
                    uniFp16SumSqr_dp8x2);\n\
                tmpSum += sumsqr.x;\n\
                sqr += (sumsqr.y * e2InScale + zpScaleSqr_i16 + zpScale2_i16 * sumsqr.x);\n\
            }\n\
            sum += tmpSum * input_scale + sumScale_i16;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = (float)(0);\n\
        sqr = (float)(0);\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        half4 tmpMean, tmpVari;\n\
        vxc_half8 tmpVal;\n\
        vxc_short8 dst;\n\
        _viv_asm(CONV, tmpMean, mean);\n\
        _viv_asm(CONV, tmpVari, vari);\n\
        VXC_DP2x8(tmpVal, tmpMean, tmpVari,\n\
                 VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
        _viv_asm(COPY, dst, tmpVal, 16);\n\
\n\
        VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of moments_axis012_vx*/

static const char moments_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int height;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float e2InScale;\n\
\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
#define MOMENTS_AXIS1_QINT(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toF16( \\\n\
    image2d_array_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    for(coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidz); \\\n\
    half4 tmpMean, tmpVari; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpMean, mean); \\\n\
    _viv_asm(CONV, tmpVari, vari); \\\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
MOMENTS_AXIS1_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS1_QINT(I8, vxc_char16)\n\
MOMENTS_AXIS1_QINT(I16, vxc_short8)\n\
\n\
#define MOMENTS_AXIS1_QINT_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toF16_2D( \\\n\
    image2d_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
        int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    for (coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
           uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, 0); \\\n\
    half4 tmpMean, tmpVari; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpMean, mean); \\\n\
    _viv_asm(CONV, tmpVari, vari); \\\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS1_QINT_2D(U8, vxc_uchar16)\n\
MOMENTS_AXIS1_QINT_2D(I8, vxc_char16)\n\
MOMENTS_AXIS1_QINT_2D(I16, vxc_short8)\n\
\n\
__kernel void moments_axis1_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 tmpSrc0;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP4x4(tmpSrc0, in_h0, in_h0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
        sum += tmpSrc0;\n\
        sqr += (tmpSrc0 * tmpSrc0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    half4 tmpMean, tmpVari;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpMean, mean);\n\
    _viv_asm(CONV, tmpVari, vari);\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void moments_axis1_F16toF16_2D(\n\
    image2d_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis, int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 tmpSrc0;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
\n\
    for(coord.y = 0; coord.y < height; coord.y++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP4x4(tmpSrc0, in_h0, in_h0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
        sum += tmpSrc0;\n\
        sqr += (tmpSrc0 * tmpSrc0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, 0);\n\
    half4 tmpMean, tmpVari;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpMean, mean);\n\
    _viv_asm(CONV, tmpVari, vari);\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of moments_axis1_vx*/

static const char moments_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
#define MOMENTS_AXIS2_QINT(src0_type_name, read0_type) \\\n\
__kernel void moments_axis2_##src0_type_name##toF16( \\\n\
    image2d_array_t input, \\\n\
    image2d_t output_mean, \\\n\
    image2d_t output_vari, \\\n\
        int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, gidy, 0, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidy); \\\n\
    half4 tmpMean, tmpVari; \\\n\
    vxc_half8 tmpVal; \\\n\
    vxc_short8 dst; \\\n\
    _viv_asm(CONV, tmpMean, mean); \\\n\
    _viv_asm(CONV, tmpVari, vari); \\\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8); \\\n\
    _viv_asm(COPY, dst, tmpVal, 16); \\\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
MOMENTS_AXIS2_QINT(U8, vxc_uchar16)\n\
MOMENTS_AXIS2_QINT(I8, vxc_char16)\n\
MOMENTS_AXIS2_QINT(I16, vxc_short8)\n\
\n\
__kernel void moments_axis2_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_t output_mean,\n\
    image2d_t output_vari,\n\
              int axis,\n\
              int axis_num)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h0;\n\
    vxc_float4 tmpSrc0;\n\
    vxc_float4 sum = (vxc_float4)(0);\n\
    vxc_float4 sqr = (vxc_float4)(0);\n\
\n\
    for(coord.z = 0; coord.z < channel; coord.z++)\n\
    {\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, in_h0, src0, 16);\n\
        VXC_DP4x4(tmpSrc0, in_h0, in_h0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), UniFP16toFP32Lo4_dp4x4);\n\
        sum += tmpSrc0;\n\
        sqr += (tmpSrc0 * tmpSrc0);\n\
    }\n\
\n\
    vxc_float4 mean = sum * dimRatio;\n\
    vxc_float4 vari = sqr * dimRatio;\n\
    vari = vari - mean * mean;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    half4 tmpMean, tmpVari;\n\
    vxc_half8 tmpVal;\n\
    vxc_short8 dst;\n\
    _viv_asm(CONV, tmpMean, mean);\n\
    _viv_asm(CONV, tmpVari, vari);\n\
    VXC_DP2x8(tmpVal, tmpMean, tmpVari, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output_mean, coord_out, dst.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output_vari, coord_out, dst.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of moments_axis2_vx*/

static const char moments_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform float4 output_ZP;\n\
_viv_uniform float4 outputScale;\n\
_viv_uniform float output_ZP0;\n\
_viv_uniform float outputScale0;\n\
_viv_uniform float output_ZP1;\n\
_viv_uniform float outputScale1;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
\n\
#define MOMENTS_AXIS0_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(0, gidy, gidz, gidz); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
    int8 inputA_desc; \\\n\
    _viv_asm(COPY, inputA_desc, input, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)gidz * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
    int2 coord_out = (int2)(gidy, gidz); \\\n\
    vxc_int4 tmpData = convert_int4_rte(mean_vari0 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
            uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS0_QINT_U8(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS0_QINT_U8_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis0_##src0_type_name##toU8_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int2 coord = (int2)(0, gidy); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    int tmpSum = 0, tmpSqr = 0; \\\n\
    int4 tmpSum0, tmpSqr0; \\\n\
 \\\n\
    for(coord.x = 0; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP16x1(tmpSum0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
        VXC_DP16x1(tmpSqr0, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
        tmpSum += (tmpSum0.x); \\\n\
        tmpSqr += (tmpSqr0.x + tmpZp1 * tmpSum0.x); \\\n\
    } \\\n\
    sqr = (convert_float(tmpSqr) * e2InScale + rowSumScale); \\\n\
    sum = convert_float(tmpSum + sumInZp) * input_scale; \\\n\
    vxc_float4 mean_vari0 = (vxc_float4)(sum, sqr, 0, 0); \\\n\
    mean_vari0 *= dimRatio; \\\n\
    mean_vari0.s1 = mean_vari0.s1 - mean_vari0.s0 * mean_vari0.s0; \\\n\
    int2 coord_out = (int2)(gidy, 0); \\\n\
    vxc_int4 tmpData = convert_int4_rte(mean_vari0 * outputScale + output_ZP); \\\n\
    VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS0_QINT_U8_2D(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS01_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    int8 inputA_desc; \\\n\
    _viv_asm(COPY, inputA_desc, input, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)gidz * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(gidz, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = (0); \\\n\
        sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 meanVari; \\\n\
        meanVari.x = sum * dimRatio; \\\n\
        meanVari.y = sqr * dimRatio - meanVari.x * meanVari.x; \\\n\
        vxc_int4 tmpData = convert_int4_rte(meanVari * outputScale + output_ZP); \\\n\
        VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT_U8(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS01_QINT_U8_2D(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis01_##src0_type_name##toU8_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
    { \\\n\
        int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.y++; \\\n\
            VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
            tmpSum += (tmpSum1); \\\n\
            VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
            tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
        } \\\n\
        sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
        sum += (tmpSum + sumInZp) * input_scale; \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        sum = (0); sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 meanVari; \\\n\
        meanVari.x = sum * dimRatio; \\\n\
        meanVari.y = sqr * dimRatio - meanVari.x * meanVari.x; \\\n\
        vxc_int4 tmpData = convert_int4_rte(meanVari * outputScale + output_ZP); \\\n\
        VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_QINT_U8_2D(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS1_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, 0, gidz, gidz); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    int8 inputA_desc; \\\n\
    _viv_asm(COPY, inputA_desc, input, sizeof(inputA_desc)); \\\n\
    int baseAddr_a = (int)gidz * inputA_desc.s4 + inputA_desc.s0; \\\n\
    _viv_asm(MOV, coord.z, baseAddr_a); \\\n\
    VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    for(coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_OP4(img_load_3d, src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio; \\\n\
    vari = vari - mean * mean; \\\n\
    vxc_int4 tmpVal0 = convert_int4_rte(mean * outputScale0 + output_ZP0); \\\n\
    vxc_int4 tmpVal1 = convert_int4_rte(vari * outputScale1 + output_ZP1); \\\n\
    VXC_DP2x8(src0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
    int2 coord_out = (int2)(gidx, gidz); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS1_QINT_U8(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS1_QINT_U8_2D(src0_type_name, read0_type) \\\n\
__kernel void moments_axis1_##src0_type_name##toU8_2D( \\\n\
    image2d_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int2 coord = (int2)(gidx, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
    VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    for (coord.y = 1; coord.y < height; ) \\\n\
    { \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
           uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    sum += (tmpData0); \\\n\
    sqr += (tmpData0 * tmpData0); \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio - mean * mean; \\\n\
    vxc_int4 tmpVal0 = convert_int4_rte(mean * outputScale0 + output_ZP0); \\\n\
    vxc_int4 tmpVal1 = convert_int4_rte(vari * outputScale1 + output_ZP1); \\\n\
    VXC_DP2x8(src0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
    int2 coord_out = (int2)(gidx, 0); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS1_QINT_U8_2D(U8, vxc_uchar16)\n\
\n\
#define MOMENTS_AXIS2_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel void moments_axis2_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int4 coord = (int4)(gidx, gidy, 0, 0); \\\n\
    read0_type src0; \\\n\
    float4 sum = 0, sqr = 0; \\\n\
    short zp = inputZP;\\\n\
    float4 tmpData0;\\\n\
 \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP4x4(tmpData0, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
                uniConvert1stUint8SubZpToFp32_4x4); \\\n\
        sum += (tmpData0); \\\n\
        sqr += (tmpData0 * tmpData0); \\\n\
    } \\\n\
    sum *= input_scale; \\\n\
    sqr *= e2InScale; \\\n\
 \\\n\
    float4 mean = sum * dimRatio; \\\n\
    float4 vari = sqr * dimRatio - mean * mean; \\\n\
    vxc_int4 tmpVal0 = convert_int4_rte(mean * outputScale0 + output_ZP0); \\\n\
    vxc_int4 tmpVal1 = convert_int4_rte(vari * outputScale1 + output_ZP1); \\\n\
    VXC_DP2x8(src0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
    int2 coord_out = (int2)(gidx, gidy); \\\n\
    VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output_vari, coord_out, src0.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
MOMENTS_AXIS2_QINT_U8(U8, vxc_uchar16)\n\
"; /* end of moments_u8_vx*/

static const char moments_u8_axis012_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int width;\n\
_viv_uniform int height;\n\
_viv_uniform int channel;\n\
_viv_uniform float dimRatio;\n\
\n\
_viv_uniform VXC_512Bits uniSumU8_16x1;\n\
_viv_uniform VXC_512Bits uniSqrSum_16x1;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int sumInZp;\n\
_viv_uniform int tmpZp1;\n\
_viv_uniform float e2InScale;\n\
_viv_uniform float rowSumScale;\n\
_viv_uniform float4 output_ZP;\n\
_viv_uniform float4 outputScale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
#define MOMENTS_AXIS012_QINT_U8(src0_type_name, read0_type) \\\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void moments_axis012_##src0_type_name##toU8( \\\n\
    image2d_array_t input, image2d_t output_mean, image2d_t output_vari, \\\n\
    int axis, int axis_num) \\\n\
{ \\\n\
    int gidx = get_global_id(0) << 4; \\\n\
    int lidx = get_local_id(0); \\\n\
    int4 coord = (int4)(gidx, 0, 0, 0); \\\n\
    read0_type src0; \\\n\
    float sum = 0, sqr = 0; \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
    for(coord.z = 0; coord.z < channel; coord.z++) \\\n\
    { \\\n\
        for(coord.x = gidx; coord.x < width; coord.x += 256) \\\n\
        { \\\n\
            int tmpSum = 0, tmpSqr = 0, tmpSum1, tmpSqr1; \\\n\
            for(coord.y = 0; coord.y < height;) \\\n\
            { \\\n\
                VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
                coord.y++; \\\n\
                VXC_DP16x1(tmpSum1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSumU8_16x1); \\\n\
                tmpSum += (tmpSum1); \\\n\
                VXC_DP16x1(tmpSqr1, src0, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniSqrSum_16x1); \\\n\
                tmpSqr += (tmpSqr1 + tmpZp1 * tmpSum1); \\\n\
            } \\\n\
            sqr += (tmpSqr * e2InScale + rowSumScale); \\\n\
            sum += (tmpSum + sumInZp) * input_scale; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
        sum = (0); sqr = (0); \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
        float4 meanVari; \\\n\
        meanVari.x = sum * dimRatio; \\\n\
        meanVari.y = sqr * dimRatio - meanVari.x * meanVari.x; \\\n\
        vxc_int4 tmpData = convert_int4_rte(meanVari * outputScale + output_ZP); \\\n\
        VXC_DP2x8(src0, tmpData, tmpData, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
                uniConvertInt32toUint8_2x8); \\\n\
        VXC_WriteImage(output_mean, coord_out, src0.s0123, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output_vari, coord_out, src0.s1023, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS012_QINT_U8(U8, vxc_uchar16)"; /* end of moments_u8_axis012_vx*/

static const char one_hot_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataConvert_0_4x4;\n\
_viv_uniform VXC_512Bits uniDataConvert_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform int depth;\n\
#define ONE_HOT_SH_IMPL(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data0, data1; \\\n\
    VXC_DP4x4(data0, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    VXC_DP4x4(data1, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_1_4x4); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        int4 d0 = data0 == coord.zzzz ? on_val : off_val; \\\n\
        int4 d1 = data1 == coord.zzzz ? on_val : off_val; \\\n\
 \\\n\
        dst_type dst; \\\n\
        VXC_DP2x8(dst, d0, d1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage2DArray(output, coord.xzyw, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        coord.z ++; \\\n\
    } while (coord.z < depth); \\\n\
}\n\
ONE_HOT_SH_IMPL(F16, F16, vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL(F16, I16, vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL(F16, I8,  vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL(F16, U8,  vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL(I16, F16, vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL(I16, I16, vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL(I8,  F16, vxc_char8,   vxc_char8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL(I8,  I8,  vxc_char8,   vxc_char8,  vxc_uchar8)\n\
\n\
#define ONE_HOT_SH_IMPL_2D(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(0), get_global_id(0)); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data, data0, data1; \\\n\
    VXC_DP4x4(data, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    int4 d4 = (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        coord.zw = coord.xx + (int2)(0, 1); \\\n\
        dst_type dst; \\\n\
        data0 = data.xxxx == d4 ? on_val : off_val; \\\n\
        data1 = data.yyyy == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord.zw = coord.zw + (int2)(2, 2); \\\n\
 \\\n\
        data0 = data.zzzz == d4 ? on_val : off_val; \\\n\
        data1 = data.wwww == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        d4 += 4; \\\n\
        coord.y += 4; \\\n\
    } while (coord.y < depth); \\\n\
}\n\
ONE_HOT_SH_IMPL_2D(F16, F16, vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(F16, I16, vxc_ushort8, vxc_half8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(F16, I8,  vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL_2D(F16, U8,  vxc_ushort8, vxc_half8,  vxc_uchar8)\n\
ONE_HOT_SH_IMPL_2D(I16, F16, vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(I16, I16, vxc_short8,  vxc_short8, vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(I8,  F16, vxc_char8,   vxc_char8,  vxc_ushort8)\n\
ONE_HOT_SH_IMPL_2D(I8,  I8,  vxc_char8,   vxc_char8,  vxc_uchar8)\n\
\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
#define ONE_HOT_ASYM_SH_IMPL(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data0, data1; \\\n\
    float4 v0, v1; \\\n\
    VXC_DP4x4(v0, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    VXC_DP4x4(v1, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_1_4x4); \\\n\
 \\\n\
    data0 = convert_int4(v0 * input_scale + input_tail); \\\n\
    data1 = convert_int4(v1 * input_scale + input_tail); \\\n\
    do \\\n\
    { \\\n\
        int4 d0 = data0 == coord.zzzz ? on_val : off_val; \\\n\
        int4 d1 = data1 == coord.zzzz ? on_val : off_val; \\\n\
 \\\n\
        dst_type dst; \\\n\
        VXC_DP2x8(dst, d0, d1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage2DArray(output, coord.xzyw, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
        coord.z ++; \\\n\
    } while (coord.z < depth); \\\n\
}\n\
ONE_HOT_ASYM_SH_IMPL(U8,  F16, vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
ONE_HOT_ASYM_SH_IMPL(U8,  U8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
\n\
#define ONE_HOT_ASYM_SH_IMPL_2D(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void one_hot_##name0##to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             suffix_sz, \\\n\
                 int             on_val, \\\n\
                 int             off_val \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(0), get_global_id(0)); \\\n\
 \\\n\
    copy_type src; \\\n\
    src_type  val; \\\n\
 \\\n\
    VXC_ReadImage(val, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, val, 16); \\\n\
 \\\n\
    int4 data, data0, data1; \\\n\
    float4 v0; \\\n\
    VXC_DP4x4(v0, src, src, \\\n\
         VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDataConvert_0_4x4); \\\n\
    int4 d4 = (int4)(0, 1, 2, 3); \\\n\
    data = convert_int4(v0 * input_scale + input_tail); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        coord.zw = coord.xx + (int2)(0, 1); \\\n\
        dst_type dst; \\\n\
        data0 = data.xxxx == d4 ? on_val : off_val; \\\n\
        data1 = data.yyyy == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord.zw = coord.zw + (int2)(2, 2); \\\n\
 \\\n\
        data0 = data.zzzz == d4 ? on_val : off_val; \\\n\
        data1 = data.wwww == d4 ? on_val : off_val; \\\n\
 \\\n\
        VXC_DP2x8(dst, data0, data1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract8Data_2x8); \\\n\
 \\\n\
        VXC_WriteImage(output, coord.yz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0)); \\\n\
        VXC_WriteImage(output, coord.yw, dst, VXC_MODIFIER(4, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        d4 += 4; \\\n\
        coord.y += 4; \\\n\
    } while (coord.y < depth); \\\n\
}\n\
ONE_HOT_ASYM_SH_IMPL_2D(U8,  F16, vxc_uchar8,  vxc_uchar8, vxc_ushort8)\n\
ONE_HOT_ASYM_SH_IMPL_2D(U8,  U8,  vxc_uchar8,  vxc_uchar8, vxc_uchar8)\n\
\n\
"; /* end of one_hot_vx*/

static const char poolwithargmax_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
//-------------------max pooling with argmax---------------\n\
_viv_uniform VXC_512Bits poolingEncode;\n\
_viv_uniform VXC_512Bits uniQuantInOutInt16Even_4x4;\n\
\n\
#define POOLWITHARGMAX_F16_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1, maxData, src0, src1; \\\n\
    vxc_half8 din0Fp16, din1Fp16; \\\n\
    vxc_half8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(src0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, src0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, src1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s02460246; \\\n\
    _viv_asm(COPY, maxData, maxDataVer, 16); \\\n\
    vxc_short8 one = (vxc_short8)(1, 1, 1, 1, 1, 1, 1, 1); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    din0EqualTmp = src0 == maxData ? one : zero; \\\n\
    din1EqualTmp = src1 == maxData ? one : zero; \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_F16to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_F16_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_F16to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_F16_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define POOLWITHARGMAX_F16_TO_I16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1, maxData, src0, src1; \\\n\
    vxc_half8 din0Fp16, din1Fp16; \\\n\
    vxc_half8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(src0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, src0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, src1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Half(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    VXC_DP4x4(din1, din0, din0, VXC_MODIFIER_BIN(0, 3, 0), uniQuantInOutInt16Even_4x4); \\\n\
    _viv_asm(COPY, maxData, maxDataVer, 16); \\\n\
    vxc_short8 one = (vxc_short8)(1, 1, 1, 1, 1, 1, 1, 1); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    din0EqualTmp = src0 == maxData ? one : zero; \\\n\
    din1EqualTmp = src1 == maxData ? one : zero; \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_F16to_I16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_F16_TO_I16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_F16to_I16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_F16_TO_I16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of poolwithargmax_F16_vx*/

static const char poolwithargmax_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits poolingEncode2;\n\
\n\
\n\
#define POOLWITHARGMAX_I16_TO_I16_U8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s02460246; \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8_SAME\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8_SAME_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniQuantInOutInt16Even_4x4;\n\
\n\
#define POOLWITHARGMAX_I16_TO_I16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    VXC_DP4x4(din1, din0, din0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniQuantInOutInt16Even_4x4); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_I16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_I16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_I16_TO_I16_I16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    vxc_short4 axisVal; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    VXC_DP4x4(din1, din0, din0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniQuantInOutInt16Even_4x4); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    axisVal = convert_short4(axisOut); \\\n\
    write_fun(axis, coordOut, axisVal, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_I16_I16\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_I16_I16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_I16_I16_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_I16_I16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirInt16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEndInt16Fp32_4x4;\n\
_viv_uniform float input_fl_scale_i16;\n\
_viv_uniform VXC_512Bits uniPackHalf8_2x8_2;\n\
\n\
#define POOLWITHARGMAX_I16_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0, din1; \\\n\
    vxc_short8 din0Fp16, din1Fp16; \\\n\
    vxc_short8 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_short8 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 din0Equal, din1Equal; \\\n\
    vxc_uchar4 axisEncode; \\\n\
    vxc_uchar4 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, din0Fp16, din0, 16); \\\n\
    _viv_asm(COPY, din1Fp16, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0Fp16, din1Fp16, din1Fp16, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s10325476; \\\n\
    _viv_asm(COPY, maxDataVer1, din1, 16); \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    _viv_asm(COPY, din0, maxDataVer, 16); \\\n\
    din1 = din0.s02460246; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    VXC_DP4x4(tmpVal0, din1, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDirInt16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, din1, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertEndInt16Fp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * input_fl_scale_i16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * input_fl_scale_i16; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackHalf8_2x8_2); \\\n\
    _viv_asm(COPY, din1, tmpPack, 16); \\\n\
    VXC_Clamp(din0EqualTmp, din0Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1Fp16, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    bitExtractCoeff = (int4)(0x30201000, 0x70605040, 0x01010101, 0x01010101); \\\n\
    VXC_BitExtract(din0Equal, din0EqualTmp, din0EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_BitExtract(din1Equal, din1EqualTmp, din1EqualTmp, bitExtractCoeff, VXC_MODIFIER_BIN(0, 7, 0)); \\\n\
    VXC_DP4x4(axisEncode, din0Equal, din1Equal, VXC_MODIFIER_BIN(0, 3, 0), poolingEncode2); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(tensorOut, coordOut, din1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I16to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I16_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I16to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I16_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of poolwithargmax_I16_vx*/

static const char poolwithargmax_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float inputScale;\n\
_viv_uniform VXC_512Bits uniPackHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniU8EvenBinSubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
_viv_uniform VXC_512Bits poolingEncodeInt8_0;\n\
_viv_uniform VXC_512Bits poolingEncodeInt8_1;\n\
\n\
#define POOLWITHARGMAX_I8_TO_I8_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 din0, din1; \\\n\
    vxc_char16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_char16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    vxc_short8 tmp; \\\n\
    short zp = input_ZP; \\\n\
    VXC_DP2x8(tmp, maxDataVer, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8EvenBinSubZP_MulM_2x8); \\\n\
    vxc_char16 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(maxDataVer1, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    write_fun(tensorOut, coordOut, maxDataVer1,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_char16)(1); \\\n\
    din1EqualTmp &= (vxc_char16)(1); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncodeInt8_0); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(4, 7, 0), poolingEncodeInt8_1); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define POOLWITHARGMAX_I8_TO_I8_U8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 din0, din1; \\\n\
    vxc_char16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_char16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s02468ace02468ace; \\\n\
    write_fun(tensorOut, coordOut, maxDataVer1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_char16)(1); \\\n\
    din1EqualTmp &= (vxc_char16)(1); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncodeInt8_0); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(4, 7, 0), poolingEncodeInt8_1); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8_SAME\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I8to_I8_U8_SAME_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I8_TO_I8_U8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertEvenU8ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEvenU8SubZpToFp32_4x4;\n\
\n\
#define POOLWITHARGMAX_I8_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 din0, din1; \\\n\
    vxc_char16 maxDataVer, maxDataVer1; \\\n\
    vxc_char16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    vxc_short8 result; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    short zp = input_ZP; \\\n\
    VXC_DP4x4(tmpVal0, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8ToFp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8SubZpToFp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniPackHalf8_2x8); \\\n\
    _viv_asm(COPY, result, tmpPack, 16); \\\n\
    write_fun(tensorOut, coordOut, result,\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_char16)(1); \\\n\
    din1EqualTmp &= (vxc_char16)(1); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 3, 0), poolingEncodeInt8_0); \\\n\
    VXC_DP4x4(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(4, 7, 0), poolingEncodeInt8_1); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void poolwithargmax_I8to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_I8_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_I8to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_I8_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of poolwithargmax_I8_vx*/

static const char poolwithargmax_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
_viv_uniform VXC_512Bits uniU8EvenBinSubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniEncodeUint8_4x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
\n\
#define POOLWITHARGMAX_U8_TO_U8_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 din0, din1; \\\n\
    vxc_uchar16 maxDataVer, maxDataVer1; \\\n\
    vxc_uchar16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    maxDataVer  = max(din0, din1); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    maxDataVer  = max(maxDataVer1, maxDataVer); \\\n\
    vxc_short8 tmp; \\\n\
    uchar zp = input_ZP; \\\n\
    VXC_DP2x8(tmp, maxDataVer, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniU8EvenBinSubZP_MulM_2x8); \\\n\
    vxc_uchar16 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(maxDataVer1, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    write_fun(tensorOut, coordOut, maxDataVer1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_uchar16)(1); \\\n\
    din1EqualTmp &= (vxc_uchar16)(1); \\\n\
    VXC_DP4x8(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 7, 0), uniEncodeUint8_4x8); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_U8_TO_U8_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_U8_TO_U8_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform VXC_512Bits uniConvertUint8ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSubZpUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniPackHalf2Short_2x8;\n\
_viv_uniform VXC_512Bits uniExtractHalf2Short_2x8;\n\
_viv_uniform VXC_512Bits uniPackHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertEvenU8ToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEvenU8SubZpToFp32_4x4;\n\
\n\
#define POOLWITHARGMAX_U8_TO_F16_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 din0, din1; \\\n\
    vxc_uchar16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_uchar16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    vxc_short8 result; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer,\\\n\
        maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    vxc_short4 tmpOut2, tmpOut3; \\\n\
    uchar zp = input_ZP; \\\n\
    VXC_DP4x4(tmpVal0, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8ToFp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, maxDataVer, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEvenU8SubZpToFp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniPackHalf8_2x8); \\\n\
    _viv_asm(COPY, result, tmpPack, 16); \\\n\
    write_fun(tensorOut, coordOut, result, \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_uchar16)(1); \\\n\
    din1EqualTmp &= (vxc_uchar16)(1); \\\n\
    VXC_DP4x8(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 7, 0), uniEncodeUint8_4x8); \\\n\
    axisOut = clz(axisEncode); \\\n\
    write_fun(axis, coordOut, axisOut, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_U8to_F16_U8\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_U8_TO_F16_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_F16_U8_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_U8_TO_F16_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_U8_TO_F16_I16_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar16 din0, din1; \\\n\
    vxc_uchar16 maxDataVer, maxDataVer1; \\\n\
    int4 bitExtractCoeff; \\\n\
    vxc_uchar16 din0EqualTmp, din1EqualTmp; \\\n\
    vxc_uchar8 axisEncode; \\\n\
    vxc_uchar8 axisOut; \\\n\
    vxc_short8 result, axisResult; \\\n\
    read_fun(din0, tensorIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(din1, tensorIn, coord, VXC_5BITOFFSET_XY(0, 1),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_VertMax3_Integer(maxDataVer, din0, din1, din1, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s1032547698badcfe; \\\n\
    VXC_VertMax3_Integer(maxDataVer, maxDataVer1, maxDataVer, maxDataVer, VXC_MODIFIER_BIN(0, 15, 0)); \\\n\
    maxDataVer1 = maxDataVer.s02468ace02468ace; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_half8 tmpPack; \\\n\
    vxc_short4 tmpOut2, tmpOut3; \\\n\
    uchar zp = input_ZP; \\\n\
    VXC_DP4x4(tmpVal0, maxDataVer1, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertUint8ToFp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, maxDataVer1, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertSubZpUint8Fp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    tmpVal3 = tmpVal2 * inputScale; \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(tmpPack, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniPackHalf8_2x8); \\\n\
    _viv_asm(COPY, result, tmpPack, 16); \\\n\
    VXC_Clamp(din0EqualTmp, din0, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    VXC_Clamp(din1EqualTmp, din1, maxDataVer, maxDataVer, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    din0EqualTmp &= (vxc_uchar16)(1); \\\n\
    din1EqualTmp &= (vxc_uchar16)(1); \\\n\
    VXC_DP4x8(axisEncode, din0EqualTmp, din1EqualTmp, VXC_MODIFIER_BIN(0, 7, 0), uniEncodeUint8_4x8); \\\n\
    axisOut = clz(axisEncode); \\\n\
    _viv_asm(CONV, axisResult, axisOut); \\\n\
    write_fun(tensorOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(axis, coordOut, axisResult, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void poolwithargmax_U8to_F16_I16\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x >> 1, coord.y >> 1, coord.z, 0);\n\
    POOLWITHARGMAX_U8_TO_F16_I16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_F16_I16_2D\n\
    (\n\
    image2d_array_t tensorIn,\n\
    image2d_array_t tensorOut,\n\
    image2d_array_t axis\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x >> 1, coord.y >> 1);\n\
    POOLWITHARGMAX_U8_TO_F16_I16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of poolwithargmax_U8_vx*/

static const char pow_fp16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalfToFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertSecUint8SubZpToFp32_4x4_2;\n\
\n\
_viv_uniform int input_ZP1;\n\
\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pow_F16F16toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_short8 dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16F16toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_short8 dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16F16toU8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_uchar8 dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16F16toU8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_uchar8 dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16U8toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0;\n\
    vxc_uchar8 src1;\n\
    vxc_short8 dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(data0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, dst, data0, 16);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16U8toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0;\n\
    vxc_uchar8 src1;\n\
    vxc_short8 dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(data0, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalfToFp16_2x8);\n\
    _viv_asm(COPY, dst, data0, 16);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16U8toU8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0;\n\
    vxc_uchar8 src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16U8toU8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0;\n\
    vxc_uchar8 src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in1_zp;\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pow_fp16_vx*/

static const char pow_fp16_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4_2;\n\
\n\
_viv_uniform float outScale_fl;\n\
\n\
__kernel void pow_F16F16toI16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16F16toI16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I16toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I16toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I16toI16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I16toI16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void pow_BF16BF16toBF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_ushort8 src0, src1, dst, tmpData;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, x0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, x1, tmpData, 16);\n\
\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, y0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, y1, tmpData, 16);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    _viv_asm(COPY, src0, tmpDst0, 16);\n\
    _viv_asm(COPY, src1, tmpDst1, 16);\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_BF16BF16toBF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_ushort8 src0, src1, dst, tmpData;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, x0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, x1, tmpData, 16);\n\
\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, y0, tmpData, 16);\n\
    VXC_DP2x8(tmpData, src1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, y1, tmpData, 16);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    _viv_asm(COPY, src0, tmpDst0, 16);\n\
    _viv_asm(COPY, src1, tmpDst1, 16);\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pow_fp16_i16_vx*/

static const char pow_fp16_i8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform float outScale_fl;\n\
\n\
__kernel void pow_F16F16toI8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_char8 dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16F16toI8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1;\n\
    vxc_char8 dst;\n\
    vxc_half8 data0, data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I8toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, dst;\n\
    vxc_char8 src1;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I8toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, dst;\n\
    vxc_char8 src1;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I8toI8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0;\n\
    vxc_char8 src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_F16I8toI8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0;\n\
    vxc_char8 src1, dst;\n\
    vxc_half8 data0;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data0, src0, 16);\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pow_fp16_i8_vx*/

static const char pow_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform float outScale_fl;\n\
\n\
__kernel void pow_I16F16toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I16F16toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I16F16toI16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I16F16toI16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I16I16toI16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I16I16toI16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pow_i16_vx*/

static const char pow_i8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform float outScale_fl;\n\
\n\
__kernel void pow_I8F16toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char8 src0;\n\
    vxc_short8 src1, dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I8F16toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_char8 src0;\n\
    vxc_short8 src1, dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I8F16toI8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char8 src0, dst;\n\
    vxc_short8 src1;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I8F16toI8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_char8 src0, dst;\n\
    vxc_short8 src1;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I8I8toI8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_char8 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_I8I8toI8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_char8 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    VXC_DP4x4(x0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outScale_fl);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outScale_fl);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pow_i8_vx*/

static const char pow_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFstDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertSecDataToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertUint8SubZpToFp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertSecUint8SubZpToFp32_4x4_2;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
_viv_uniform int input_ZP0;\n\
_viv_uniform int input_ZP1;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pow_U8F16toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 src0;\n\
    vxc_short8 src1;\n\
    vxc_short8 dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_U8F16toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar8 src0;\n\
    vxc_short8 src1;\n\
    vxc_short8 dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_U8F16toU8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 src0;\n\
    vxc_short8 src1;\n\
    vxc_uchar8 dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_U8F16toU8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar8 src0;\n\
    vxc_short8 src1;\n\
    vxc_uchar8 dst;\n\
    vxc_half8 data1;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, data1, src1, 16);\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstDataToFp32_4x4_2);\n\
    VXC_DP4x4(y1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecDataToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_U8U8toU8(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 src0, src1, dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_U8U8toU8_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar8 src0, src1, dst;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4_2);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4_2);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    int4 tmpVal0 = convert_int4_rte(tmpDst0 * outputScale + output_ZP);\n\
    int4 tmpVal1 = convert_int4_rte(tmpDst1 * outputScale + output_ZP);\n\
    VXC_DP2x8(dst, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\\\n\
            uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_U8U8toF16(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    vxc_uchar8 src0;\n\
    vxc_uchar8 src1;\n\
    vxc_short8 dst;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage2DArray(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    vxc_half8 tmpVal;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(tmpVal, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pow_U8U8toF16_2D(\n\
    image2d_array_t input0,\n\
    image2d_array_t input1,\n\
    image2d_array_t output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_uchar8 src0;\n\
    vxc_uchar8 src1;\n\
    vxc_short8 dst;\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    float4 x0, x1;\n\
    float4 y0, y1;\n\
    float4 tmpDst0, tmpDst1;\n\
    short in0_zp, in1_zp;\n\
    _viv_asm(COPY, in0_zp, input_ZP0, 4);\n\
    _viv_asm(COPY, in1_zp, input_ZP1, 4);\n\
    VXC_DP4x4(x0, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(x1, src0, in0_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y0, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertUint8SubZpToFp32_4x4);\n\
    VXC_DP4x4(y1, src1, in1_zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecUint8SubZpToFp32_4x4);\n\
\n\
    float4  s0 = sign(x0);\n\
    float4  s1 = sign(x1);\n\
    int4 t0 = convert_int4(y0) & 1;\n\
    int4 t1 = convert_int4(y1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    s1 = s1 == -1 ? convert_float4(t1) == 1.0f ? -1.0f : 1.0f : s1;\n\
    tmpDst0 = s0 * exp2(y0 * log2(fabs(x0)));\n\
    tmpDst1 = s1 * exp2(y1 * log2(fabs(x1)));\n\
\n\
    half4 tmpVal0, tmpVal1;\n\
    vxc_half8 tmpVal;\n\
    _viv_asm(CONV, tmpVal0, tmpDst0);\n\
    _viv_asm(CONV, tmpVal1, tmpDst1);\n\
    VXC_DP2x8(tmpVal, tmpVal0, tmpVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pow_u8_vx*/

static const char pre_process_bgra_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1Bgra_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2Bgra_4x4;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtractInt32BgraToU8_2x8;\n\
_viv_uniform VXC_512Bits uniExchangeBgra_2x8;\n\
_viv_uniform VXC_512Bits uniExchangeBgra2_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp5BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp6BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp7BgraShort_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp8BgraShort_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniExtractBfromBgra_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGfromBgra_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRfromBgra_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_bgra_scale_U8toU8(\n\
    __read_only image2d_array_t input, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx = (sx + (*xOffset)) * 4 ;\n\
    sy += (*yOffset);\n\
    int4 srcPos = (int4)(sx.x, sy, sy + 1, sx.y);\n\
    vxc_uchar16 lineBGRA0, lineBGRA1, lineBGRA2, lineBGRA3;\n\
    vxc_uchar16 dataB, dataG, dataR;\n\
\n\
    VXC_ReadImage(lineBGRA0, input, srcPos.xy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA0, input, srcPos.xz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(lineBGRA1, input, srcPos.wy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA1, input, srcPos.wz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    srcPos.w = sx.w;\n\
\n\
    VXC_ReadImage(lineBGRA2, input, srcPos.xy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA2, input, srcPos.xz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(lineBGRA3, input, srcPos.wy, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lineBGRA3, input, srcPos.wz, VXC_5BITOFFSET_XY(0, 0),\n\
                     VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar4 val_u8;\n\
    int4 tmp1, tmp2, result1, result2;\n\
    float4 tmpDst, tmp0;\n\
    float4 mean = (float4)(bMean, gMean, rMean, 0);\n\
    //tmpFx = (int4)(fx.x, fx.x, fx.x, fx.x);\n\
    int tmpV = 1 << 19;\n\
    vxc_short8 tmpFx;\n\
    VXC_DP2x8(tmpFx, fx, fx, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    //tmpFx = fx.xxxx;\n\
    VXC_DP4x4(tmp1, lineBGRA0, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA0, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    //tmpFx = fx.yyyy;\n\
    VXC_DP4x4(tmp1, lineBGRA1, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA1, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result2 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    vxc_uchar16 dst, data;\n\
    VXC_DP2x8(dst, result1, result2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInt32BgraToU8_2x8);\n\
\n\
    //tmpFx = fx.zzzz;\n\
    VXC_DP4x4(tmp1, lineBGRA2, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp5BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA2, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp6BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    //tmpFx = fx.wwww;\n\
    VXC_DP4x4(tmp1, lineBGRA3, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp7BgraShort_4x4);\n\
    VXC_DP4x4(tmp2, lineBGRA3, tmpFx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp8BgraShort_4x4);\n\
    tmp1 = fy * (tmp2 - tmp1) + (tmp1 << 10);\n\
    VXC_DP4x4(val_u8, tmp1, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    VXC_DP4x4(tmp0, val_u8, val_u8, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertIntergetoF32_4x4);\n\
    tmpDst = (tmp0 - mean) * var;\n\
    result2 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    VXC_DP2x8(dst, result1, result2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtractInt32BgraToU8_2x8);\n\
\n\
    VXC_DP2x8(data, dst, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExchangeBgra_2x8);\n\
    VXC_DP2x8(data, dst, dst, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0), uniExchangeBgra2_2x8);\n\
\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    dstPos.z = bOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    dstPos.z = 1;\n\
    VXC_WriteImage2DArray(output, dstPos, data.s4567, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    dstPos.z = rOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, data.s89ab, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_bgra_copy_U8toU8(\n\
    __read_only image2d_array_t input, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int2 pos = (int2)((get_global_id(0) + (*xOffset)) << 2, get_global_id(1) + (*yOffset));\n\
\n\
    vxc_uchar16 lineBGRA0;\n\
    float4 tmpB, tmpG, tmpR;\n\
    float4 tmpDst;\n\
    int4 result1, result2;\n\
    vxc_uchar16 dst;\n\
\n\
    VXC_ReadImage(lineBGRA0, input, pos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x4(tmpB, lineBGRA0, lineBGRA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBfromBgra_4x4);\n\
    VXC_DP4x4(tmpG, lineBGRA0, lineBGRA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGfromBgra_4x4);\n\
    VXC_DP4x4(tmpR, lineBGRA0, lineBGRA0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRfromBgra_4x4);\n\
\n\
    tmpDst = (tmpB - bMean) * var;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
\n\
    tmpDst = (tmpG - gMean) * var;\n\
    result2 = convert_int4_rte(tmpDst * outputScale + zp);\n\
    VXC_DP2x8(dst, result1, result2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
\n\
    int4 dstPos = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    dstPos.z = bOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    dstPos.z = 1;\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(4, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    tmpDst = (tmpR - rMean) * var;\n\
    result1 = convert_int4_rte(tmpDst * outputScale + zp);\n\
    VXC_DP2x8(dst, result1, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
\n\
    dstPos.z = rOrder;\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_bgra_vx*/

static const char pre_process_gray_vx[] = "/*\n\
 ============================================================================\n\
 Name        : GrayScale.vx\n\
 Author      : Sam\n\
 Version     :\n\
 Copyright   : Your copyright notice\n\
 Description :\n\
 ============================================================================\n\
 */\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform VXC_512Bits uniExtactInteger_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniDataMulAlpha_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubMean_4x4;\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
\n\
__kernel void pre_process_gray_scale_U8toF16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio);\n\
\n\
    int4 xPos       = get_global_id(0);\n\
    int yPos        = get_global_id(1);\n\
\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14);\n\
    xPos += (int4)(0, 1, 2, 3);\n\
\n\
    //x\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x;\n\
    int4 sx = fx0 & 0xffff8000;\n\
    fx0 -= sx;\n\
    sx = sx >> 15;\n\
\n\
    vxc_short4 fx;\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift);\n\
    //y\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y;\n\
    int sy = fy & 0xffff8000; // Floor\n\
\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
\n\
    //R\n\
    vxc_uchar16 line0Y;\n\
    vxc_uchar16 line1Y;\n\
    int4 coord;\n\
    sx = sx + *xOffset;\n\
    coord.xyz    = sx.xyz;\n\
    coord.w        = sy + *yOffset;\n\
    int2 coord1 = (int2)(sx.w, coord.w);\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float grayMean = mean;\n\
\n\
    int4 test01, temp1;\n\
    int4 test02, temp2;\n\
    int4 tt;\n\
    vxc_uchar4 val;\n\
    int2 coord_out = (int2)(xPos.x, yPos);\n\
\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniGetTempVal);\n\
    temp1 = temp1 + test01;\n\
\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniGetTempVal);\n\
    temp2 = temp2 + test02;\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10);\n\
\n\
    VXC_DP4x4(val, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\n\
        uniExtractBytes);\n\
\n\
    //convert U8 to FP16\n\
    half f16mean;\n\
    half f16alpha;\n\
    vxc_half4    dst;\n\
    vxc_short4 tmp_dst;\n\
    _viv_asm(CONV, f16mean, grayMean);\n\
    _viv_asm(CONV, f16alpha, f32Var);\n\
    VXC_DP4x4(dst, val, f16mean, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniDataSubMean_4x4);\n\
    VXC_DP4x4(dst, dst, f16alpha, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniDataMulAlpha_4x4);\n\
    _viv_asm(COPY, tmp_dst, dst, 8);\n\
    VXC_WriteImage(output, coord_out, tmp_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_gray_scale_U8toI16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio);\n\
\n\
    int4 xPos        = get_global_id(0);\n\
    int yPos        = get_global_id(1);\n\
\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14);\n\
    xPos += (int4)(0, 1, 2, 3);\n\
\n\
    //x\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x;\n\
    int4 sx = fx0 & 0xffff8000;\n\
    fx0 -= sx;\n\
    sx = sx >> 15;\n\
\n\
    vxc_short4 fx;\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniAddRShift);\n\
    //y\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y;\n\
    int sy = fy & 0xffff8000; // Floor\n\
\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
\n\
    vxc_uchar16 line0Y;\n\
    vxc_uchar16 line1Y;\n\
    int4 coord;\n\
    sx = sx + *xOffset;\n\
    coord.xyz    = sx.xyz;\n\
    coord.w        = sy + *yOffset;\n\
    int2 coord1 = (int2)(sx.w, coord.w);\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1),\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float grayMean = mean * f32Var;\n\
\n\
    int4 test01, temp1;\n\
    int4 test02, temp2;\n\
    int4 tt;\n\
    vxc_uchar4 val;\n\
    int2 coord_out = (int2)(xPos.x, yPos);\n\
\n\
    vxc_uchar8 line1, line2;\n\
\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal);\n\
    temp1 = temp1 + test01;\n\
\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\n\
        uniVecShift10);\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal);\n\
    temp2 = temp2 + test02;\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10);\n\
\n\
    vxc_float4    tmp_dst;\n\
    vxc_uchar4 u8_dst;\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\n\
        uniExtractBytes);\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1),\n\
        uniConvertIntergetoF32_4x4);\n\
\n\
    //convert U8 to dfp8\n\
    int4 dst0;\n\
    vxc_short4 dst;\n\
    tmp_dst = tmp_dst * f32Var - grayMean;\n\
    tmp_dst = tmp_dst * outputScale  + outputZP;\n\
    dst0 = convert_int4_rte(tmp_dst);\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),\n\
        uniExtactInteger_2x8);\n\
\n\
    VXC_WriteImage(output, coord_out, dst,\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define PRE_PROCESS_GRAY_SCALE_8BITS(dst_type_name, write_type) \\\n\
__kernel void pre_process_gray_scale_U8to##dst_type_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
        global int               *xRatio, \\\n\
        global int               *yRatio, \\\n\
        global int               *xOffset, \\\n\
        global int               *yOffset, \\\n\
               float             mean, \\\n\
               float             f32Var \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos        = get_global_id(0); \\\n\
    int yPos        = get_global_id(1); \\\n\
 \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
 \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0Y; \\\n\
    vxc_uchar16 line1Y; \\\n\
    int4 coord; \\\n\
    sx = sx + *xOffset; \\\n\
    coord.xyz    = sx.xyz; \\\n\
    coord.w        = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0Y, input, coord.xw, 0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.yw, 0, VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord.zw, 0, VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0Y, input, coord1, 0, VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1Y, input, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1Y, input, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float grayMean = mean * f32Var; \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int2 coord_out = (int2)(xPos.x, yPos); \\\n\
 \\\n\
    VXC_DP4x4(test01, line0Y, line0Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line0Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line1Y, line1Y, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line1Y, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    int4 dst0; \\\n\
    write_type dst; \\\n\
    tmp_dst = tmp_dst * f32Var - grayMean; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    dst0 = convert_int4_rte(tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniExtactInteger_2x8); \\\n\
 \\\n\
    VXC_WriteImage(output, coord_out, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
PRE_PROCESS_GRAY_SCALE_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_SCALE_8BITS(I8, vxc_char16)"; /* end of pre_process_gray_vx*/

static const char pre_process_gray_2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l00_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l10_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l01_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l11_4x4;\n\
_viv_uniform VXC_512Bits uniBilinear_4over3_l21_4x4;\n\
\n\
__kernel void pre_process_gray_4over3_U8toU8\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar16 src0, src1, src2, src3;\n\
\n\
    VXC_ReadImage(src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src1, input, coord_in, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src2, input, coord_in, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(src3, input, coord_in, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.xy = (coord_in.xy >> 2) * 3;\n\
    coord_in.zw = coord_in.yy + (int2)(1, 2);\n\
\n\
    vxc_uchar16 dst0, dst1, dst2;\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l00_2x8);\n\
    VXC_DP2x8(dst0, src0, src0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l10_2x8);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst1, src1, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l01_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniBilinear_4over3_l11_4x4);\n\
    VXC_DP4x4(dst2, src3, src2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniBilinear_4over3_l21_4x4);\n\
\n\
    VXC_WriteImage(output, coord_in.xy, dst0, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_in.xz, dst1, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_in.xw, dst2, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_gray_half_U8toU8\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
\n\
    vxc_uchar16 src0;\n\
\n\
    VXC_ReadImage(src0, input, coord_in, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_in.xy = coord_in.xy >> 1;\n\
\n\
    VXC_WriteImage(output, coord_in.xy, src0.s02468ace, VXC_MODIFIER(0, 11, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_gray_2_vx*/

static const char pre_process_gray_copy_vx[] = "/*\n\
 ============================================================================\n\
 Name        : GrayScale.vx\n\
 Author      : Sam\n\
 Version     :\n\
 Copyright   : Your copyright notice\n\
 Description :\n\
 ============================================================================\n\
 */\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniDataMeanStddevLo_2x8;\n\
_viv_uniform VXC_512Bits uniDataMeanStddevHi_2x8;\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
\n\
__kernel void pre_process_gray_copy_U8toF16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    coord.xy += (int2) (*xOffset, *yOffset);\n\
    vxc_uchar16 src0;\n\
    vxc_half8   dst0, dst1;\n\
\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.x = coord.z + 8;\n\
    float4      paramData = (float4)(mean * f32Var, mean * f32Var, mean * f32Var, f32Var);\n\
    //convert U8 to FP16\n\
    half4 paramData_f16;\n\
    vxc_short8 tmp_dst;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
        uniDataMeanStddevLo_2x8);\n\
    VXC_DP2x8(dst1, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
        uniDataMeanStddevHi_2x8);\n\
    _viv_asm(COPY, tmp_dst, dst0, 16);\n\
    VXC_WriteImage(output, coord.zw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, tmp_dst, dst1, 16);\n\
    VXC_WriteImage(output, coord.xw, tmp_dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_gray_copy_U8toI16\n\
    (\n\
    __read_only image2d_array_t  input,\n\
    __write_only image2d_array_t output,\n\
        global int               *xRatio,\n\
        global int               *yRatio,\n\
        global int               *xOffset,\n\
        global int               *yOffset,\n\
               float             mean,\n\
               float             f32Var\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
\n\
    coord.xy += (int2) (*xOffset, *yOffset);\n\
    vxc_uchar16 src0;\n\
    vxc_short8  dst0, dst1;\n\
\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord.x = coord.z + 8;\n\
\n\
    f32Var *= outputScale;\n\
    float4 paramData = (float4)(mean * f32Var - outputZP, mean * f32Var - outputZP,\n\
                         mean * f32Var - outputZP, f32Var);\n\
    //convert U8 to FP16\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
\n\
    VXC_DP2x8(dst0, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniDataMeanStddevLo_2x8);\n\
    VXC_DP2x8(dst1, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\n\
        uniDataMeanStddevHi_2x8);\n\
    VXC_WriteImage(output, coord.zw, dst0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord.xw, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define PRE_PROCESS_GRAY_COPY_8BITS(dst_type_name, write_type) \\\n\
__kernel void pre_process_gray_copy_U8to##dst_type_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
        global int               *xRatio, \\\n\
        global int               *yRatio, \\\n\
        global int               *xOffset, \\\n\
        global int               *yOffset, \\\n\
               float             mean, \\\n\
               float             f32Var \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    coord.xy += (int2) (*xOffset, *yOffset); \\\n\
    vxc_uchar16 src0; \\\n\
    write_type dst; \\\n\
 \\\n\
    VXC_ReadImage(src0, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    f32Var *= outputScale; \\\n\
    float4  paramData = (float4)(mean * f32Var - outputZP, mean * f32Var - outputZP, \\\n\
        mean * f32Var - outputZP, f32Var); \\\n\
 \\\n\
    half4 paramData_f16; \\\n\
    _viv_asm(CONV, paramData_f16, paramData); \\\n\
 \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevLo_2x8); \\\n\
    VXC_DP2x8(dst, src0, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniDataMeanStddevHi_2x8); \\\n\
    VXC_WriteImage(output, coord.zw, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
PRE_PROCESS_GRAY_COPY_8BITS(U8, vxc_uchar16)\n\
PRE_PROCESS_GRAY_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_gray_copy_vx*/

static const char pre_process_nv12_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar;\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform uint xrIntFloat_16;\n\
_viv_uniform uint yrIntFloat_16;\n\
\n\
_viv_uniform VXC_512Bits uniConvertNV12toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUVtoCharSub128_2x8;\n\
\n\
__kernel void pre_process_nv12_scale_U8toI16(\n\
    __read_only image2d_t y_img, __read_only image2d_t uv_img,\n\
    __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    uint4 gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    gidx += (uint4)(0, 1, 2, 3);\n\
\n\
    uint dy = (gidy * yrIntFloat_16) >> 16;\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16;\n\
    int sy = convert_int(dy) + (*yOffset);\n\
    int4 sx = convert_int4(dx) + (*xOffset);\n\
    int4 uvX = sx & 0xfffffffe;\n\
    int uvY = sy >> 1;\n\
\n\
    vxc_uchar16 Y, UV;\n\
    int2 coord = (int2)(sx.x, sy);\n\
    int2 coord_uv = (int2)(uvX.x, uvY);\n\
\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.y;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.z;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.w;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.y;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.z;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.w;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_char16 tmpUV;\n\
    short tmpVal = 128;\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8);\n\
\n\
    float4 tmpDstB, tmpDstG, tmpDstR;\n\
    VXC_DP4x4(tmpDstB, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toB_4x4);\n\
    VXC_DP4x4(tmpDstG, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toG_4x4);\n\
    VXC_DP4x4(tmpDstR, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toR_4x4);\n\
\n\
    int4 result;\n\
    vxc_short8 dst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    result = convert_int4_rte(tmpDstB * outputScaleVar + bMeanScaleVarZp);\n\
    dstPos.z = bOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstG * outputScaleVar + gMeanScaleVarZp);\n\
    dstPos.z = 1;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstR * outputScaleVar + rMeanScaleVarZp);\n\
    dstPos.z = rOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_nv12_scale_U8toF16(\n\
    __read_only image2d_t y_img, __read_only image2d_t uv_img,\n\
    __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    uint4 gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    gidx += (uint4)(0, 1, 2, 3);\n\
\n\
    uint dy = (gidy * yrIntFloat_16) >> 16;\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16;\n\
    int sy = convert_int(dy) + (*yOffset);\n\
    int4 sx = convert_int4(dx) + (*xOffset);\n\
    int4 uvX = sx & 0xfffffffe;\n\
    int uvY = sy >> 1;\n\
\n\
    vxc_uchar16 Y, UV;\n\
    int2 coord = (int2)(sx.x, sy);\n\
    int2 coord_uv = (int2)(uvX.x, uvY);\n\
\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.y;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.z;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.w;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.y;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.z;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.w;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_char16 tmpUV;\n\
    short tmpVal = 128;\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8);\n\
\n\
    float4 tmpDstB, tmpDstG, tmpDstR;\n\
    VXC_DP4x4(tmpDstB, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toB_4x4);\n\
    VXC_DP4x4(tmpDstG, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toG_4x4);\n\
    VXC_DP4x4(tmpDstR, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toR_4x4);\n\
\n\
    tmpDstB = tmpDstB * outputScaleVar + bMeanScaleVarZp;\n\
    tmpDstG = tmpDstG * outputScaleVar + gMeanScaleVarZp;\n\
    tmpDstR = tmpDstR * outputScaleVar + rMeanScaleVarZp;\n\
\n\
    half4 result;\n\
    vxc_half8 tmpdst;\n\
    vxc_short8 dst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    _viv_asm(CONV, result, tmpDstB);\n\
    dstPos.z = bOrder;\n\
    VXC_DP2x8(tmpdst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpdst, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(CONV, result, tmpDstG);\n\
    dstPos.z = 1;\n\
    VXC_DP2x8(tmpdst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpdst, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(CONV, result, tmpDstR);\n\
    dstPos.z = rOrder;\n\
    VXC_DP2x8(tmpdst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpdst, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_nv12_scale_vx*/

static const char pre_process_nv12_scale_8bits_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar;\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform uint  xrIntFloat_16;\n\
_viv_uniform uint  yrIntFloat_16;\n\
\n\
_viv_uniform VXC_512Bits uniConvertNV12toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUVtoCharSub128_2x8;\n\
_viv_uniform VXC_512Bits uniExtractUVtoCharSub128_2x8;\n\
\n\
__kernel void pre_process_nv12_scale_U8toU8(\n\
    __read_only image2d_t y_img, __read_only image2d_t uv_img,\n\
    __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    uint4 gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    gidx += (uint4)(0, 1, 2, 3);\n\
\n\
    uint dy = (gidy * yrIntFloat_16) >> 16;\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16;\n\
    int sy = convert_int(dy) + (*yOffset);\n\
    int4 sx = convert_int4(dx) + (*xOffset);\n\
    int4 uvX = sx & 0xfffffffe;\n\
    int uvY = sy >> 1;\n\
\n\
    vxc_uchar16 Y, UV;\n\
    int2 coord = (int2)(sx.x, sy);\n\
    int2 coord_uv = (int2)(uvX.x, uvY);\n\
\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.y;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.z;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.w;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.y;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.z;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.w;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_char16 tmpUV;\n\
    short tmpVal = 128;\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8);\n\
\n\
    float4 tmpDstB, tmpDstG, tmpDstR;\n\
    VXC_DP4x4(tmpDstB, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toB_4x4);\n\
    VXC_DP4x4(tmpDstG, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toG_4x4);\n\
    VXC_DP4x4(tmpDstR, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toR_4x4);\n\
\n\
    int4 result;\n\
    vxc_uchar8 dst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    result = convert_int4_rte(tmpDstB * outputScaleVar + bMeanScaleVarZp);\n\
    dstPos.z = bOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstG * outputScaleVar + gMeanScaleVarZp);\n\
    dstPos.z = 1;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstR * outputScaleVar + rMeanScaleVarZp);\n\
    dstPos.z = rOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_nv12_copy_U8toU8(\n\
    __read_only image2d_t y_img, __read_only image2d_t uv_img,\n\
    __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int sy = gidy + (*yOffset);\n\
    int sx = gidx + (*xOffset);\n\
    int uvX = sx & 0xfffffffe;\n\
    int uvY = sy >> 1;\n\
\n\
    vxc_uchar16 Y, UV;\n\
\n\
    VXC_ReadImage(Y, y_img, (int2)(sx,sy), VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(UV, uv_img,(int2)(uvX,uvY), VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_char16 tmpUV;\n\
    short tmpVal = 128;\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractUVtoCharSub128_2x8);\n\
\n\
    float4 tmpDstB, tmpDstG, tmpDstR;\n\
    VXC_DP4x4(tmpDstB, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toB_4x4);\n\
    VXC_DP4x4(tmpDstG, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toG_4x4);\n\
    VXC_DP4x4(tmpDstR, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toR_4x4);\n\
\n\
    int4 result;\n\
    vxc_uchar8 dst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    result = convert_int4_rte(tmpDstB * outputScaleVar + bMeanScaleVarZp);\n\
    dstPos.z = bOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstG * outputScaleVar + gMeanScaleVarZp);\n\
    dstPos.z = 1;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstR * outputScaleVar + rMeanScaleVarZp);\n\
    dstPos.z = rOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_nv12_scale_U8toI8(\n\
    __read_only image2d_t y_img, __read_only image2d_t uv_img,\n\
    __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    uint4 gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    gidx += (uint4)(0, 1, 2, 3);\n\
\n\
    uint dy = (gidy * yrIntFloat_16) >> 16;\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16;\n\
    int sy = convert_int(dy) + (*yOffset);\n\
    int4 sx = convert_int4(dx) + (*xOffset);\n\
    int4 uvX = sx & 0xfffffffe;\n\
    int uvY = sy >> 1;\n\
\n\
    vxc_uchar16 Y, UV;\n\
    int2 coord = (int2)(sx.x, sy);\n\
    int2 coord_uv = (int2)(uvX.x, uvY);\n\
\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.y;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.z;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord.x = sx.w;\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.y;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.z;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_uv.x = uvX.w;\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_char16 tmpUV;\n\
    short tmpVal = 128;\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8);\n\
\n\
    float4 tmpDstB, tmpDstG, tmpDstR;\n\
    VXC_DP4x4(tmpDstB, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toB_4x4);\n\
    VXC_DP4x4(tmpDstG, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toG_4x4);\n\
    VXC_DP4x4(tmpDstR, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toR_4x4);\n\
\n\
    int4 result;\n\
    vxc_char8 dst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    result = convert_int4_rte(tmpDstB * outputScaleVar + bMeanScaleVarZp);\n\
    dstPos.z = bOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstG * outputScaleVar + gMeanScaleVarZp);\n\
    dstPos.z = 1;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstR * outputScaleVar + rMeanScaleVarZp);\n\
    dstPos.z = rOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_nv12_scale_8bits_vx*/

static const char pre_process_nv12_scale_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
_viv_uniform float outputScaleVar;\n\
_viv_uniform float bMeanScaleVarZp;\n\
_viv_uniform float gMeanScaleVarZp;\n\
_viv_uniform float rMeanScaleVarZp;\n\
\n\
_viv_uniform uint  xrIntFloat_16;\n\
_viv_uniform uint  yrIntFloat_16;\n\
\n\
_viv_uniform VXC_512Bits uniConvertNV12toB_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toG_4x4;\n\
_viv_uniform VXC_512Bits uniConvertNV12toR_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniConvertUVtoCharSub128_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateYShift_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateUVShift_2x8;\n\
\n\
__kernel void pre_process_nv12_scale_U8toU8_gq(\n\
    __read_only image2d_t y_img, __read_only image2d_t uv_img,\n\
    __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    uint4 gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    gidx += (uint4)(0, 1, 2, 3);\n\
\n\
    uint dy = (gidy * yrIntFloat_16) >> 16;\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16;\n\
    int sy = convert_int(dy) + (*yOffset);\n\
    int4 sx = convert_int4(dx) + (*xOffset);\n\
    int4 uvX = sx & 0xfffffffe;\n\
    int uvY = sy >> 1;\n\
\n\
    vxc_uchar16 Y, UV;\n\
    int2 coord = (int2)(sx.x, sy);\n\
    int2 coord_uv = (int2)(uvX.x, uvY);\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    vxc_uchar16 maskShiftUv = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    int4 offsetUV = uvX - uvX.x;\n\
\n\
    vxc_ushort8 diffY, diffUV;\n\
    _viv_asm(COPY, diffY, sx, 16);\n\
    _viv_asm(COPY, diffUV, offsetUV, 16);\n\
\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, diffY, constData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniCalculateYShift_2x8);\n\
    VXC_DP2x8(maskShiftUv, diffUV, constData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniCalculateUVShift_2x8);\n\
    VXC_BitExtract(Y, Y, Y, maskShift, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(UV, UV, UV, maskShiftUv, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_char16 tmpUV;\n\
    short tmpVal = 128;\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8);\n\
\n\
    float4 tmpDstB, tmpDstG, tmpDstR;\n\
    VXC_DP4x4(tmpDstB, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toB_4x4);\n\
    VXC_DP4x4(tmpDstG, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toG_4x4);\n\
    VXC_DP4x4(tmpDstR, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toR_4x4);\n\
\n\
    int4 result;\n\
    vxc_uchar8 dst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    result = convert_int4_rte(tmpDstB * outputScaleVar + bMeanScaleVarZp);\n\
    dstPos.z = bOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstG * outputScaleVar + gMeanScaleVarZp);\n\
    dstPos.z = 1;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    result = convert_int4_rte(tmpDstR * outputScaleVar + rMeanScaleVarZp);\n\
    dstPos.z = rOrder;\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_nv12_scale_U8toF16_gq(\n\
    __read_only image2d_t y_img, __read_only image2d_t uv_img,\n\
    __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    uint4 gidx = get_global_id(0);\n\
    uint gidy = get_global_id(1);\n\
    gidx += (uint4)(0, 1, 2, 3);\n\
\n\
    uint dy = (gidy * yrIntFloat_16) >> 16;\n\
    uint4 dx = (gidx * xrIntFloat_16) >> 16;\n\
    int sy = convert_int(dy) + (*yOffset);\n\
    int4 sx = convert_int4(dx) + (*xOffset);\n\
    int4 uvX = sx & 0xfffffffe;\n\
    int uvY = sy >> 1;\n\
\n\
    vxc_uchar16 Y, UV;\n\
    int2 coord = (int2)(sx.x, sy);\n\
    int2 coord_uv = (int2)(uvX.x, uvY);\n\
    VXC_ReadImage(Y, y_img, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(UV, uv_img,coord_uv, VXC_5BITOFFSET_XY(0,0),VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    vxc_uchar16 maskShiftUv = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    int4 offsetUV = uvX - uvX.x;\n\
\n\
    vxc_ushort8 diffY, diffUV;\n\
    _viv_asm(COPY, diffY, sx, 16);\n\
    _viv_asm(COPY, diffUV, offsetUV, 16);\n\
\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, diffY, constData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniCalculateYShift_2x8);\n\
    VXC_DP2x8(maskShiftUv, diffUV, constData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniCalculateUVShift_2x8);\n\
    VXC_BitExtract(Y, Y, Y, maskShift, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(UV, UV, UV, maskShiftUv, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_char16 tmpUV;\n\
    short tmpVal = 128;\n\
    VXC_DP2x8(tmpUV, UV, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertUVtoCharSub128_2x8);\n\
\n\
    float4 tmpDstB, tmpDstG, tmpDstR;\n\
    VXC_DP4x4(tmpDstB, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toB_4x4);\n\
    VXC_DP4x4(tmpDstG, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toG_4x4);\n\
    VXC_DP4x4(tmpDstR, Y, tmpUV, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertNV12toR_4x4);\n\
\n\
    tmpDstB = tmpDstB * outputScaleVar + bMeanScaleVarZp;\n\
    tmpDstG = tmpDstG * outputScaleVar + gMeanScaleVarZp;\n\
    tmpDstR = tmpDstR * outputScaleVar + rMeanScaleVarZp;\n\
\n\
    half4 result;\n\
    vxc_half8 tmpdst;\n\
    vxc_short8 dst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    _viv_asm(CONV, result, tmpDstB);\n\
    dstPos.z = bOrder;\n\
    VXC_DP2x8(tmpdst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpdst, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(CONV, result, tmpDstG);\n\
    dstPos.z = 1;\n\
    VXC_DP2x8(tmpdst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpdst, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    _viv_asm(CONV, result, tmpDstR);\n\
    dstPos.z = rOrder;\n\
    VXC_DP2x8(tmpdst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpdst, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_nv12_scale_mix_vx*/

static const char pre_process_rgb_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniVecShift10;\n\
_viv_uniform VXC_512Bits uniAddRShift;\n\
_viv_uniform VXC_512Bits uniGetTempVal;\n\
_viv_uniform VXC_512Bits uniExtractBytes;\n\
_viv_uniform VXC_512Bits uniUnpackToR;\n\
_viv_uniform VXC_512Bits uniUnpackToG;\n\
_viv_uniform VXC_512Bits uniUnpackToB;\n\
\n\
_viv_uniform VXC_512Bits uniConvertIntergetoF32_4x4;\n\
_viv_uniform float outputScale;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float outputZP;\n\
_viv_uniform int r_order;\n\
_viv_uniform int b_order;\n\
\n\
#define DESCALE(x) (((x) + (1<<19)) >> 20)\n\
\n\
#define IMAGE_PRE_PROCESS(dst_name, conv_type, dst_type, copy_type) \\\n\
__kernel void pre_process_rgb_scale_U8to##dst_name \\\n\
    ( \\\n\
__read_only image2d_array_t  input, \\\n\
__write_only image2d_array_t output, \\\n\
        global int           *xRatio, \\\n\
        global int           *yRatio, \\\n\
        global int           *xOffset, \\\n\
        global int           *yOffset, \\\n\
               float         rMean, \\\n\
               float         gMean, \\\n\
               float         bMean, \\\n\
               float         f32Var, \\\n\
               int           reverse_channel, \\\n\
               int           trans \\\n\
    ) \\\n\
{ \\\n\
    int2 ratioXY = (int2)(*xRatio, *yRatio); \\\n\
    int4 xPos       = get_global_id(0); \\\n\
    int yPos        = get_global_id(1); \\\n\
    int2 ratioSufXY = (ratioXY >> 1) - (1 << 14); \\\n\
    xPos += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    /*x*/ \\\n\
    int4 fx0 = xPos * ratioXY.x + ratioSufXY.x; \\\n\
    int4 sx = fx0 & 0xffff8000; \\\n\
    fx0 -= sx; \\\n\
    sx = sx >> 15; \\\n\
 \\\n\
    vxc_short4 fx; \\\n\
    VXC_DP4x4(fx, fx0, 1 << 4, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniAddRShift); \\\n\
    /*y*/ \\\n\
    int fy = yPos * ratioXY.y + ratioSufXY.y; \\\n\
    int sy = fy & 0xffff8000; \\\n\
 \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
 \\\n\
    vxc_uchar16 line0RGB1, line0RGB2; \\\n\
    vxc_uchar16 line1RGB3, line1RGB4; \\\n\
    int4 coord; \\\n\
    sx = (sx + (*xOffset)) * 3; \\\n\
    coord.xyz    = sx.xyz; \\\n\
    coord.w        = sy + *yOffset; \\\n\
    int2 coord1 = (int2)(sx.w, coord.w); \\\n\
    VXC_ReadImage(line0RGB1, input, coord.xw, 0, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0RGB1, input, coord.yw, 0, VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0RGB2, input, coord.zw, 0, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line0RGB2, input, coord1, 0, VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_ReadImage(line1RGB3, input, coord.xw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1RGB3, input, coord.yw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1RGB4, input, coord.zw, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(line1RGB4, input, coord1, VXC_5BITOFFSET_XY(0, 1), \\\n\
        VXC_MODIFIER(6, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    float4 bgrMean = (float4)(bMean, gMean, rMean, 0); \\\n\
 \\\n\
    bgrMean *= f32Var; \\\n\
 \\\n\
    int4 test01, temp1; \\\n\
    int4 test02, temp2; \\\n\
    int4 tt; \\\n\
    vxc_uchar4 val; \\\n\
    int4 coord_out = (int4)(xPos.x, yPos, r_order, 0); \\\n\
 \\\n\
    vxc_uchar8 line1, line2; \\\n\
 \\\n\
    /*R*/ \\\n\
    VXC_DP2x8(line1, line0RGB1, line0RGB2, \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToR); \\\n\
    VXC_DP2x8(line2, line1RGB3, line1RGB4, \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToR); \\\n\
 \\\n\
    VXC_DP4x4(test01, line1, line1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line1, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line2, line2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line2, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    vxc_float4 tmp_dst; \\\n\
    vxc_uchar4 u8_dst; \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    /*convert U8 to dst*/ \\\n\
    dst_type dst; \\\n\
    tmp_dst = tmp_dst * f32Var - bgrMean.zzzz; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    conv_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    copy_type result; \\\n\
    _viv_asm(COPY, result, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, result, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    /*G*/ \\\n\
    VXC_DP2x8(line1, line0RGB1, line0RGB2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToG); \\\n\
    VXC_DP2x8(line2, line1RGB3, line1RGB4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToG); \\\n\
 \\\n\
    coord_out.z = 1; \\\n\
    VXC_DP4x4(test01, line1, line1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line1, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line2, line2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line2, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
            uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * f32Var - bgrMean.y; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, result, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, result, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    /*B*/ \\\n\
    VXC_DP2x8(line1, line0RGB1, line0RGB2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToB); \\\n\
    VXC_DP2x8(line2, line1RGB3, line1RGB4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniUnpackToB); \\\n\
 \\\n\
    coord_out.z = b_order; \\\n\
    VXC_DP4x4(test01, line1, line1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp1, line1, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp1 = temp1 + test01; \\\n\
 \\\n\
    VXC_DP4x4(test02, line2, line2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniVecShift10); \\\n\
    VXC_DP4x4(temp2, line2, fx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetTempVal); \\\n\
    temp2 = temp2 + test02; \\\n\
    temp2 = fy * (temp2 - temp1) + (temp1 << 10); \\\n\
 \\\n\
    VXC_DP4x4(u8_dst, temp2, 1 << 19, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), uniExtractBytes); \\\n\
    VXC_DP4x4(tmp_dst, u8_dst, u8_dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
        uniConvertIntergetoF32_4x4); \\\n\
 \\\n\
    tmp_dst = tmp_dst * f32Var - bgrMean.x; \\\n\
    tmp_dst = tmp_dst * outputScale + outputZP; \\\n\
    _viv_asm(CONV_RTE, dst0, tmp_dst); \\\n\
    VXC_DP2x8(dst, dst0, dst0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, result, dst, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, result, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS(U8,  uint4, vxc_uchar16, vxc_uchar16)\n\
IMAGE_PRE_PROCESS(I8,  int4,  vxc_char16,  vxc_char16)\n\
IMAGE_PRE_PROCESS(I16, int4,  vxc_short8,  vxc_short8)\n\
IMAGE_PRE_PROCESS(F16, half4, vxc_half8,   vxc_short8)\n\
"; /* end of pre_process_rgb_vx*/

static const char pre_process_rgb_copy_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform int r_order;\n\
_viv_uniform int b_order;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractRtoF32_part3_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractGtoF32_part3_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractBtoF32_part3_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
\n\
#define IMAGE_PRE_PROCESS_COPY_16BITS(dst_name, dst_type, copy_type, convert_type) \\\n\
__kernel void pre_process_rgb_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
         global int              *xRatio, \\\n\
         global int              *yRatio, \\\n\
         global int              *xOffset, \\\n\
         global int              *yOffset, \\\n\
                float            rMean, \\\n\
                float            gMean, \\\n\
                float            bMean, \\\n\
                float            f32Var, \\\n\
                int              reverse_channel, \\\n\
                int              trans \\\n\
    ) \\\n\
{ \\\n\
    int2 coord      = (int2)(get_global_id(0) * 3, get_global_id(1)); \\\n\
 \\\n\
    coord.xy = coord.xy + (int2) (*xOffset * 3 + 16, *yOffset); \\\n\
    vxc_uchar16 src0, src1; \\\n\
    dst_type   dst0; \\\n\
    copy_type   dst; \\\n\
 \\\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(-16, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    f32Var *= outputScale; \\\n\
    float4 paramData = (float4)(rMean * f32Var - outputZP, gMean * f32Var - outputZP, \\\n\
        bMean * f32Var - outputZP, f32Var); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), r_order, 0); \\\n\
    float4 tmp0, tmp1; \\\n\
    convert_type result0, result1; \\\n\
 \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.x; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.x; \\\n\
    _viv_asm(CONV_RTE, result0, tmp0); \\\n\
    _viv_asm(CONV_RTE, result1, tmp1); \\\n\
    VXC_DP2x8(dst0, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = 1; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.y; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.y; \\\n\
    _viv_asm(CONV_RTE, result0, tmp0); \\\n\
    _viv_asm(CONV_RTE, result1, tmp1); \\\n\
    VXC_DP2x8(dst0, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = b_order; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.z; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.z; \\\n\
    _viv_asm(CONV_RTE, result0, tmp0); \\\n\
    _viv_asm(CONV_RTE, result1, tmp1); \\\n\
    VXC_DP2x8(dst0, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS_COPY_16BITS(I16, vxc_short8, vxc_short8, int4)\n\
IMAGE_PRE_PROCESS_COPY_16BITS(F16, vxc_half8,  vxc_short8, half4)\n\
\n\
#define IMAGE_PRE_PROCESS_COPY_8BITS(dst_name, dst_type) \\\n\
__kernel void pre_process_rgb_copy_U8to##dst_name \\\n\
    ( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
         global int              *xRatio, \\\n\
         global int              *yRatio, \\\n\
         global int              *xOffset, \\\n\
         global int              *yOffset, \\\n\
                float            rMean, \\\n\
                float            gMean, \\\n\
                float            bMean, \\\n\
                float            f32Var, \\\n\
                int              reverse_channel, \\\n\
                int              trans \\\n\
    ) \\\n\
{ \\\n\
    int2 coord      = (int2)(get_global_id(0) * 3, get_global_id(1)); \\\n\
    coord.xy = coord.xy + (int2) (*xOffset * 3 + 16, *yOffset); \\\n\
    vxc_uchar16 src0, src1, src2; \\\n\
    dst_type dst; \\\n\
 \\\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(-16, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(src1, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.x += 16; \\\n\
    VXC_ReadImage(src2, input, coord.xy, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    f32Var *= outputScale; \\\n\
    float4 paramData = (float4)(rMean * f32Var - outputZP, gMean * f32Var - outputZP, \\\n\
        bMean * f32Var - outputZP, f32Var); \\\n\
 \\\n\
    int4 coord_out = (int4)(get_global_id(0), get_global_id(1), r_order, 0); \\\n\
    float4 tmp0, tmp1; \\\n\
    int4 result0, result1; \\\n\
 \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.x; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.x; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_DP4x4(tmp0, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part2_4x4); \\\n\
    VXC_DP4x4(tmp1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractRtoF32_part3_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.x; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.x; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = 1; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.y; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.y; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_DP4x4(tmp0, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part2_4x4); \\\n\
    VXC_DP4x4(tmp1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractGtoF32_part3_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.y; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.y; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    coord_out.z = b_order; \\\n\
    VXC_DP4x4(tmp0, src0, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part0_4x4); \\\n\
    VXC_DP4x4(tmp1, src0, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part1_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.z; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.z; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_DP4x4(tmp0, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part2_4x4); \\\n\
    VXC_DP4x4(tmp1, src1, src2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractBtoF32_part3_4x4); \\\n\
    tmp0 = tmp0 * paramData.w - paramData.z; \\\n\
    tmp1 = tmp1 * paramData.w - paramData.z; \\\n\
    result0 = convert_int4_rte(tmp0); \\\n\
    result1 = convert_int4_rte(tmp1); \\\n\
    VXC_DP2x8(dst, result0, result1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS_COPY_8BITS(U8, vxc_uchar16)\n\
IMAGE_PRE_PROCESS_COPY_8BITS(I8, vxc_char16)\n\
"; /* end of pre_process_rgb_copy_vx*/

static const char pre_process_yuv420_copy_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpB1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoR_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiR_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_yuv420_copy_U8toU8(\n\
    __read_only image2d_t            y_img,\n\
    __read_only image2d_t            u_img,\n\
    __read_only image2d_t            v_img,\n\
    __write_only image2d_array_t    output,\n\
        global int *                xRatio,\n\
        global int *                yRatio,\n\
        global int *               xOffset,\n\
        global int *               yOffset,\n\
               float                 rMean,\n\
               float                 gMean,\n\
               float                 bMean,\n\
               float                   var,\n\
               int         reverse_channel,\n\
               int                   trans\n\
    )\n\
{\n\
    int4 pos = (int4)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset), 0, 0);\n\
    int2 pos1 = (int2)((get_global_id(0) + (*xOffset)) >> 1, (get_global_id(1) + (*yOffset)) >> 1);\n\
    vxc_uchar16 Y;\n\
    vxc_uchar8 U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
    vxc_uchar16 dst0, dst1, dst2;\n\
\n\
    VXC_ReadImage(Y, y_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, pos1.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, pos1.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16;\n\
    //D = U - 128;\n\
    //E = V - 128;\n\
    // calculate R\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4);\n\
\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // calculate G\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG3rd_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG4th_4x4);\n\
\n\
    // calculate B\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    var *= outputScale;\n\
    float4  paramData = (float4)(bMean * var - zp, gMean * var - zp,\\\n\
        rMean * var - zp, var);\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8);\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8);\n\
\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8);\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8);\n\
\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8);\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8);\n\
\n\
    pos = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    pos.z = bOrder;\n\
    VXC_WriteImage2DArray(output, pos, dst0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    pos.z = 1;\n\
    VXC_WriteImage2DArray(output, pos, dst1, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    pos.z = rOrder;\n\
    VXC_WriteImage2DArray(output, pos, dst2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_yuv420_copy_U8toF16(\n\
    __read_only image2d_t            y_img,\n\
    __read_only image2d_t            u_img,\n\
    __read_only image2d_t            v_img,\n\
    __write_only image2d_array_t    output,\n\
        global int *                xRatio,\n\
        global int *                yRatio,\n\
        global int *               xOffset,\n\
        global int *               yOffset,\n\
               float                 rMean,\n\
               float                 gMean,\n\
               float                 bMean,\n\
               float                   var,\n\
               int         reverse_channel,\n\
               int                   trans\n\
    )\n\
{\n\
    int4 pos = (int4)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset), 0, 0);\n\
    int2 pos1 = (int2)((get_global_id(0) + (*xOffset)) >> 1, (get_global_id(1) + (*yOffset)) >> 1);\n\
    vxc_uchar16 Y;\n\
    vxc_uchar8 U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
    vxc_half8 dst0, dst1, dst2, dst3, dst4, dst5;\n\
    vxc_short8 out0, out1, out2, out3, out4, out5;\n\
\n\
    VXC_ReadImage(Y, y_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, pos1.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, pos1.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16;\n\
    //D = U - 128;\n\
    //E = V - 128;\n\
    // calculate R\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4);\n\
\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // calculate G\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG3rd_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG4th_4x4);\n\
\n\
    // calculate B\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    float4  paramData = (float4)(bMean * var, gMean * var,\\\n\
        rMean * var, var);\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8);\n\
    VXC_DP2x8(dst1, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8);\n\
\n\
    VXC_DP2x8(dst2, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8);\n\
    VXC_DP2x8(dst3, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8);\n\
\n\
    VXC_DP2x8(dst4, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8);\n\
    VXC_DP2x8(dst5, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8);\n\
\n\
    _viv_asm(COPY, out0, dst0, 16);\n\
    _viv_asm(COPY, out1, dst1, 16);\n\
    _viv_asm(COPY, out2, dst2, 16);\n\
    _viv_asm(COPY, out3, dst3, 16);\n\
    _viv_asm(COPY, out4, dst4, 16);\n\
    _viv_asm(COPY, out5, dst5, 16);\n\
\n\
    pos = (int4)(get_global_id(0), get_global_id(1), bOrder, get_global_id(0) + 8);\n\
    VXC_WriteImage2DArray(output, pos.xyzz, out0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos.wyzz, out1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    pos.z = 1;\n\
    VXC_WriteImage2DArray(output, pos.xyzz, out2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos.wyzz, out3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    pos.z = rOrder;\n\
    VXC_WriteImage2DArray(output, pos.xyzz, out4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos.wyzz, out5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of pre_process_yuv420_copy_u8_vx*/

static const char pre_process_yuv420_scale_fp16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
__kernel void pre_process_yuv420_scale_U8toF16(\n\
    __read_only image2d_array_t y_img, __read_only image2d_array_t u_img,\n\
    __read_only image2d_array_t v_img, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx += (*xOffset);\n\
    sy += (*yOffset);\n\
    int4 srcPos = (int4)(sx.x, sy, get_global_id(2), 0);\n\
    int4 srcPos1 = (int4)(sx.x >> 1, sy >> 1, get_global_id(2), 0);\n\
    int4 srcPos2 = (int4)(sx.x >> 1, (sy + 1) >> 1, get_global_id(2), 0);\n\
\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.y;\n\
    srcPos1.x = sx.y >> 1;\n\
    srcPos2.x = sx.y >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    srcPos1.x = sx.z >> 1;\n\
    srcPos2.x = sx.z >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.w;\n\
    srcPos1.x = sx.w >> 1;\n\
    srcPos2.x = sx.w >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16; D = U - 128; E = V - 128;\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4);\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    int4 result, temp1, temp2;\n\
    int4 tmpData0, tmpData1;\n\
\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    // temp2 - temp1\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
\n\
    vxc_half8 tmpVal;\n\
    half4 hDst;\n\
    tmpV = 1 << 19;\n\
    vxc_short8 dst;\n\
    float4 tmpDst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - bMean) * var;\n\
    dstPos.z = bOrder;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - gMean) * var;\n\
    dstPos.z = 1;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - rMean) * var;\n\
    dstPos.z = rOrder;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_yuv420_scale_fp16_vx*/

static const char pre_process_yuv420_scale_i16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_yuv420_scale_U8toI16(\n\
    __read_only image2d_array_t y_img, __read_only image2d_array_t u_img,\n\
    __read_only image2d_array_t v_img, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx += (*xOffset);\n\
    sy += (*yOffset);\n\
    int4 srcPos = (int4)(sx.x, sy, get_global_id(2), 0);\n\
    int4 srcPos1 = (int4)(sx.x >> 1, sy >> 1, get_global_id(2), 0);\n\
    int4 srcPos2 = (int4)(sx.x >> 1, (sy + 1) >> 1, get_global_id(2), 0);\n\
\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.y;\n\
    srcPos1.x = sx.y >> 1;\n\
    srcPos2.x = sx.y >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    srcPos1.x = sx.z >> 1;\n\
    srcPos2.x = sx.z >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.w;\n\
    srcPos1.x = sx.w >> 1;\n\
    srcPos2.x = sx.w >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16; D = U - 128; E = V - 128;\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4);\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    int4 result, temp1, temp2;\n\
    int4 tmpData0, tmpData1;\n\
\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    // temp2 - temp1\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
\n\
    tmpV = 1 << 19;\n\
    vxc_short8 dst;\n\
    float4 tmpDst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - bMean) * var;\n\
    dstPos.z = bOrder;\n\
    result = convert_int4_rte(tmpDst * outputScale);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - gMean) * var;\n\
    dstPos.z = 1;\n\
    result = convert_int4_rte(tmpDst * outputScale);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - rMean) * var;\n\
    dstPos.z = rOrder;\n\
    result = convert_int4_rte(tmpDst * outputScale);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_yuv420_scale_i16_vx*/

static const char pre_process_yuv420_scale_i8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_yuv420_scale_U8toI8(\n\
    __read_only image2d_array_t y_img, __read_only image2d_array_t u_img,\n\
    __read_only image2d_array_t v_img, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx += (*xOffset);\n\
    sy += (*yOffset);\n\
    int4 srcPos = (int4)(sx.x, sy, get_global_id(2), 0);\n\
    int4 srcPos1 = (int4)(sx.x >> 1, sy >> 1, get_global_id(2), 0);\n\
    int4 srcPos2 = (int4)(sx.x >> 1, (sy + 1) >> 1, get_global_id(2), 0);\n\
\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.y;\n\
    srcPos1.x = sx.y >> 1;\n\
    srcPos2.x = sx.y >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    srcPos1.x = sx.z >> 1;\n\
    srcPos2.x = sx.z >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.w;\n\
    srcPos1.x = sx.w >> 1;\n\
    srcPos2.x = sx.w >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16; D = U - 128; E = V - 128;\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4);\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    int4 result, temp1, temp2;\n\
    int4 tmpData0, tmpData1;\n\
\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    // temp2 - temp1\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
\n\
    tmpV = 1 << 19;\n\
    vxc_char8 dst;\n\
    float4 tmpDst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - bMean) * var;\n\
    dstPos.z = bOrder;\n\
    result = convert_int4_rte(tmpDst * outputScale);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - gMean) * var;\n\
    dstPos.z = 1;\n\
    result = convert_int4_rte(tmpDst * outputScale);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - rMean) * var;\n\
    dstPos.z = rOrder;\n\
    result = convert_int4_rte(tmpDst * outputScale);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_yuv420_scale_i8_vx*/

static const char pre_process_yuv420_scale_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_yuv420_scale_U8toU8(\n\
    __read_only image2d_array_t y_img, __read_only image2d_array_t u_img,\n\
    __read_only image2d_array_t v_img, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx += (*xOffset);\n\
    sy += (*yOffset);\n\
    int4 srcPos = (int4)(sx.x, sy, get_global_id(2), 0);\n\
    int4 srcPos1 = (int4)(sx.x >> 1, sy >> 1, get_global_id(2), 0);\n\
    int4 srcPos2 = (int4)(sx.x >> 1, (sy + 1) >> 1, get_global_id(2), 0);\n\
\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.x + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.y;\n\
    srcPos1.x = sx.y >> 1;\n\
    srcPos2.x = sx.y >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 4, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(5, 5, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(6, 6, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.y + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(7, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    srcPos1.x = sx.z >> 1;\n\
    srcPos2.x = sx.z >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 8, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(9, 9, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(10, 10, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.z + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(11, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.w;\n\
    srcPos1.x = sx.w >> 1;\n\
    srcPos2.x = sx.w >> 1;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 12, 0, VXC_RM_TowardZero, 0));\n\
    srcPos1.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos1, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(13, 13, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(14, 14, 0, VXC_RM_TowardZero, 0));\n\
    srcPos2.x = (sx.w + 1) >> 1;\n\
    VXC_ReadImage(U, u_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos2, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(15, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16; D = U - 128; E = V - 128;\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4);\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    int4 result, temp1, temp2;\n\
    int4 tmpData0, tmpData1;\n\
\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    // temp2 - temp1\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
\n\
    tmpV = 1 << 19;\n\
    vxc_uchar8 dst;\n\
    float4 tmpDst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - bMean) * var;\n\
    dstPos.z = bOrder;\n\
    result = convert_int4_rte(tmpDst * outputScale + zp);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - gMean) * var;\n\
    dstPos.z = 1;\n\
    result = convert_int4_rte(tmpDst * outputScale + zp);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - rMean) * var;\n\
    dstPos.z = rOrder;\n\
    result = convert_int4_rte(tmpDst * outputScale + zp);\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_yuv420_scale_u8_vx*/

static const char pre_process_yuv444_copy_u8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpR4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpG4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateG1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateG2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpB1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpB4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiB_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiG_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8LoR_2x8;\n\
_viv_uniform VXC_512Bits uniQuantU8toU8HiR_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
__kernel void pre_process_yuv444_copy_U8toU8(\n\
    __read_only image2d_t            y_img,\n\
    __read_only image2d_t            u_img,\n\
    __read_only image2d_t            v_img,\n\
    __write_only image2d_array_t    output,\n\
        global int *                xRatio,\n\
        global int *                yRatio,\n\
        global int *               xOffset,\n\
        global int *               yOffset,\n\
               float                 rMean,\n\
               float                 gMean,\n\
               float                 bMean,\n\
               float                   var,\n\
               int         reverse_channel,\n\
               int                   trans\n\
    )\n\
{\n\
    int2 pos = (int2)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset));\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
    vxc_uchar16 dst0, dst1, dst2;\n\
\n\
    VXC_ReadImage(Y, y_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16;\n\
    //D = U - 128;\n\
    //E = V - 128;\n\
    // calculate R\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4);\n\
\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // calculate G\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG0, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG0, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2_2x8);\n\
\n\
    VXC_DP4x4(G, C0, tmpDstG0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
\n\
    // calculate B\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    var *= outputScale;\n\
    float4  paramData = (float4)(bMean * var - zp, gMean * var - zp,\\\n\
        rMean * var - zp, var);\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8);\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8);\n\
\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8);\n\
    VXC_DP2x8(dst1, G, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8);\n\
\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8);\n\
    VXC_DP2x8(dst2, R, paramData_f16, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8);\n\
\n\
    int4 pos1 = (int4)(get_global_id(0), get_global_id(1), bOrder, 0);\n\
    VXC_WriteImage2DArray(output, pos1, dst0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = 1;\n\
    VXC_WriteImage2DArray(output, pos1, dst1, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = rOrder;\n\
    VXC_WriteImage2DArray(output, pos1, dst2, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void pre_process_yuv444_copy_U8toF16(\n\
    __read_only image2d_t            y_img,\n\
    __read_only image2d_t            u_img,\n\
    __read_only image2d_t            v_img,\n\
    __write_only image2d_array_t    output,\n\
        global int *                xRatio,\n\
        global int *                yRatio,\n\
        global int *               xOffset,\n\
        global int *               yOffset,\n\
               float                 rMean,\n\
               float                 gMean,\n\
               float                 bMean,\n\
               float                   var,\n\
               int         reverse_channel,\n\
               int                   trans\n\
    )\n\
{\n\
    int2 pos = (int2)(get_global_id(0) + (*xOffset), get_global_id(1) + (*yOffset));\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
    vxc_half8 dst0, dst1, dst2, dst3, dst4, dst5;\n\
    vxc_short8 out0, out1, out2, out3, out4, out5;\n\
\n\
    VXC_ReadImage(Y, y_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, pos.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16;\n\
    //D = U - 128;\n\
    //E = V - 128;\n\
    // calculate R\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpR4th_4x4);\n\
\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // calculate G\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG1st_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpG4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG0, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG0, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2_2x8);\n\
\n\
    VXC_DP4x4(G, C0, tmpDstG0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateG1st_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateG2nd_4x4);\n\
\n\
    // calculate B\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB1st_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpB4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    float4  paramData = (float4)(bMean * var, gMean * var,\\\n\
        rMean * var, var);\n\
    half4 paramData_f16;\n\
    _viv_asm(CONV, paramData_f16, paramData);\n\
\n\
    VXC_DP2x8(dst0, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoB_2x8);\n\
    VXC_DP2x8(dst1, B, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiB_2x8);\n\
\n\
    VXC_DP2x8(dst2, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoG_2x8);\n\
    VXC_DP2x8(dst3, G, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiG_2x8);\n\
\n\
    VXC_DP2x8(dst4, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8LoR_2x8);\n\
    VXC_DP2x8(dst5, R, paramData_f16, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantU8toU8HiR_2x8);\n\
\n\
    _viv_asm(COPY, out0, dst0, 16);\n\
    _viv_asm(COPY, out1, dst1, 16);\n\
    _viv_asm(COPY, out2, dst2, 16);\n\
    _viv_asm(COPY, out3, dst3, 16);\n\
    _viv_asm(COPY, out4, dst4, 16);\n\
    _viv_asm(COPY, out5, dst5, 16);\n\
\n\
    int4 pos1 = (int4)(get_global_id(0), get_global_id(1), bOrder, get_global_id(0) + 8);\n\
    VXC_WriteImage2DArray(output, pos1.xyzz, out0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos1.wyzz, out1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = 1;\n\
    VXC_WriteImage2DArray(output, pos1.xyzz, out2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos1.wyzz, out3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    pos1.z = rOrder;\n\
    VXC_WriteImage2DArray(output, pos1.xyzz, out4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage2DArray(output, pos1.wyzz, out5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of pre_process_yuv444_copy_u8_vx*/

static const char pre_process_yuv444_scale_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
_viv_uniform int zp;\n\
_viv_uniform float outputScale;\n\
\n\
#define IMAGE_PRE_PROCESS_YUV444_QINT(dst_name, dst_type) \\\n\
__kernel void pre_process_yuv444_scale_U8to##dst_name( \\\n\
    __read_only image2d_t y_img, __read_only image2d_t u_img, \\\n\
    __read_only image2d_t v_img, __write_only image2d_array_t    output, \\\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset, \\\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans) \\\n\
{ \\\n\
    int4 gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    gidx += (int4)(0, 1, 2, 3); \\\n\
 \\\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14); \\\n\
    int4 sx = fx & 0xffff8000;  \\\n\
    int fy, sy; \\\n\
    fx -= sx; \\\n\
    sx = sx >> 15; \\\n\
    fx = (fx +(1 << 4)) >> 5; \\\n\
 \\\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14); \\\n\
    sy = fy & 0xffff8000;  \\\n\
    fy -= sy; \\\n\
    sy = sy >> 15; \\\n\
 \\\n\
    sy = sy < 0 ? 0 : sy; \\\n\
    fy = fy < 0 ? 0 : fy; \\\n\
 \\\n\
    fy = (fy + (1<< 4)) >> 5; \\\n\
    sx += (*xOffset); \\\n\
    sy += (*yOffset); \\\n\
    int2 srcPos = (int2)(sx.x, sy); \\\n\
 \\\n\
    vxc_uchar16 Y, U, V; \\\n\
    vxc_int4 C0, C1, C2, C3; \\\n\
    vxc_uchar16 R, G, B; \\\n\
 \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.y; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.z; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    srcPos.x = sx.w; \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int tmpV = -56992; \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4); \\\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4); \\\n\
 \\\n\
    ushort tmpG = 34784; \\\n\
    vxc_ushort8 tmpDstG, tmpDstG1; \\\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8); \\\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8); \\\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4); \\\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4); \\\n\
 \\\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4); \\\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4); \\\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4); \\\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4); \\\n\
    tmpV = -70688; \\\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4); \\\n\
 \\\n\
    int4 result, temp1, temp2; \\\n\
    int4 tmpData0, tmpData1; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
 \\\n\
    tmpV = 1 << 19; \\\n\
    dst_type dst; \\\n\
    float4 tmpDst; \\\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - bMean) * var; \\\n\
    dstPos.z = bOrder; \\\n\
    result = convert_int4_rte(tmpDst * outputScale + zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - gMean) * var; \\\n\
    dstPos.z = 1; \\\n\
    result = convert_int4_rte(tmpDst * outputScale + zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4); \\\n\
    temp1 = fx * tmpData0 + tmpData1; \\\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4); \\\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4); \\\n\
    temp2 = fx * tmpData0 + tmpData1; \\\n\
    result = fy * temp2 + (temp1 << 10); \\\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4); \\\n\
    tmpDst = (tmpDst - rMean) * var; \\\n\
    dstPos.z = rOrder; \\\n\
    result = convert_int4_rte(tmpDst * outputScale + zp); \\\n\
    VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
IMAGE_PRE_PROCESS_YUV444_QINT(U8, vxc_uchar8)\n\
IMAGE_PRE_PROCESS_YUV444_QINT(I8, vxc_char8)\n\
IMAGE_PRE_PROCESS_YUV444_QINT(I16, vxc_short8)"; /* end of pre_process_yuv444_scale_vx*/

static const char pre_process_yuv444_scale_fp16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniCalculateR1st_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU_2x8;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGbyU2nd_2x8;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateB1st_4x4;\n\
_viv_uniform VXC_512Bits uniDescaleU8_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpRWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpGWise4th_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateGWise2nd_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise2nd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise3rd_4x4;\n\
_viv_uniform VXC_512Bits uniCalculateTmpBWise4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniBilinearTmp1st_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp2nd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp3rd_4x4;\n\
_viv_uniform VXC_512Bits uniBilinearTmp4th_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertHalftoFp16_2x8;\n\
\n\
_viv_uniform int bOrder;\n\
_viv_uniform int rOrder;\n\
\n\
__kernel void pre_process_yuv444_scale_U8toF16(\n\
    __read_only image2d_t y_img, __read_only image2d_t u_img,\n\
    __read_only image2d_t v_img, __write_only image2d_array_t    output,\n\
    global int *xRatio, global int * yRatio, global int * xOffset, global int * yOffset,\n\
    float rMean, float gMean, float bMean, float var, int reverse_channel, int trans)\n\
{\n\
    int4 gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    gidx += (int4)(0, 1, 2, 3);\n\
\n\
    int4 fx = (gidx * (*xRatio) + ((*xRatio) >> 1)) - (1 << 14);\n\
    int4 sx = fx & 0xffff8000; // Floor\n\
    int fy, sy;\n\
    fx -= sx;\n\
    sx = sx >> 15;\n\
    fx = (fx +(1 << 4)) >> 5;\n\
\n\
    // for y\n\
    fy = (gidy * (*yRatio) + ((*yRatio) >> 1)) - (1<< 14);\n\
    sy = fy & 0xffff8000; // Floor\n\
    fy -= sy;\n\
    sy = sy >> 15;\n\
\n\
    sy = sy < 0 ? 0 : sy;\n\
    fy = fy < 0 ? 0 : fy;\n\
\n\
    fy = (fy + (1<< 4)) >> 5;\n\
    sx += (*xOffset);\n\
    sy += (*yOffset);\n\
    int2 srcPos = (int2)(sx.x, sy);\n\
\n\
    vxc_uchar16 Y, U, V;\n\
    vxc_int4 C0, C1, C2, C3;\n\
    vxc_uchar16 R, G, B;\n\
\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.y;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.z;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(8, 9, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(10, 11, 0, VXC_RM_TowardZero, 0));\n\
\n\
    srcPos.x = sx.w;\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(Y, y_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(U, u_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(12, 13, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(V, v_img, srcPos, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(14, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //C = Y - 16; D = U - 128; E = V - 128;\n\
    // ((298 * C + 409 * E + 128) >> 8) -->  [(298Y + 409V - 56992) >> 8]\n\
    int tmpV = -56992;\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpRWise4th_4x4);\n\
    VXC_DP4x4(R, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
    VXC_DP4x4(R, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateR1st_4x4);\n\
\n\
    // ((298 * C - 100* D - 208 * E + 128) >> 8) --> [(298Y - 100U - 208V + 34784) >> 8]\n\
    // 298Y - 208V\n\
    VXC_DP4x4(C0, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise_4x4);\n\
    VXC_DP4x4(C1, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, V, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGWise4th_4x4);\n\
    // 34784 - 100U\n\
    ushort tmpG = 34784;\n\
    vxc_ushort8 tmpDstG, tmpDstG1;\n\
    VXC_DP2x8(tmpDstG, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU_2x8);\n\
    VXC_DP2x8(tmpDstG1, U, tmpG, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniCalculateTmpGbyU2nd_2x8);\n\
    VXC_DP4x4(G, C0, tmpDstG, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C1, tmpDstG, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
    VXC_DP4x4(G, C2, tmpDstG1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise_4x4);\n\
    VXC_DP4x4(G, C3, tmpDstG1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateGWise2nd_4x4);\n\
\n\
    // ((298 * C + 516 * D + 128) >> 8) ==> [(298Y + 516U - 70688) >> 8]\n\
    VXC_DP4x4(C0, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise_4x4);\n\
    VXC_DP4x4(C1, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise2nd_4x4);\n\
    VXC_DP4x4(C2, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise3rd_4x4);\n\
    VXC_DP4x4(C3, Y, U, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniCalculateTmpBWise4th_4x4);\n\
    tmpV = -70688;\n\
    VXC_DP4x4(B, C0, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C1, tmpV, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C2, tmpV, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
    VXC_DP4x4(B, C3, tmpV, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniCalculateB1st_4x4);\n\
\n\
    int4 result, temp1, temp2;\n\
    int4 tmpData0, tmpData1;\n\
\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    // temp2 - temp1\n\
    VXC_DP4x4(tmpData0, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, B, B, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
\n\
    vxc_half8 tmpVal;\n\
    half4 hDst;\n\
    tmpV = 1 << 19;\n\
    vxc_short8 dst;\n\
    float4 tmpDst;\n\
    int4 dstPos = (int4)(get_global_id(0), gidy, 0, 0);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - bMean) * var;\n\
    dstPos.z = bOrder;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, G, G, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - gMean) * var;\n\
    dstPos.z = 1;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp1st_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp2nd_4x4);\n\
    temp1 = fx * tmpData0 + tmpData1;\n\
    VXC_DP4x4(tmpData0, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp3rd_4x4);\n\
    VXC_DP4x4(tmpData1, R, R, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniBilinearTmp4th_4x4);\n\
    temp2 = fx * tmpData0 + tmpData1;\n\
    result = fy * temp2 + (temp1 << 10);\n\
    VXC_DP4x4(tmpDst, result, tmpV, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniDescaleU8_4x4);\n\
    tmpDst = (tmpDst - rMean) * var;\n\
    dstPos.z = rOrder;\n\
    _viv_asm(CONV, hDst, tmpDst);\n\
    VXC_DP2x8(tmpVal, hDst, hDst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertHalftoFp16_2x8);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, dstPos, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of pre_process_yuv444_scale_fp16_vx*/

static const char prelu_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
#if (VX_VERSION==2)\n\
_viv_uniform VXC_512Bits uniPreluDFPLo_2x8b;\n\
_viv_uniform VXC_512Bits uniPreluDFPHi_2x8b;\n\
__kernel void prelu_I8F16toI8_2D_OPT\n\
(\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_char16 in, dst;\n\
    vxc_char32 src;\n\
    vxc_short8 a0, a1;\n\
    vxc_half8 c0, c1;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a1, param, coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    _viv_asm(COPY, c1, a1, 4);\n\
    src.hi = max(in, 0);\n\
    src.lo = min(in, 0);\n\
\n\
    VXC_DP2x8_b(dst, src.hi, src.lo, c0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluDFPLo_2x8b);\n\
    VXC_DP2x8_b(dst, src.hi, src.lo, c1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniPreluDFPHi_2x8b);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void prelu_I16F16toI16_2D_OPT\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    vxc_short8 in, dst;\n\
    vxc_short16 src;\n\
    vxc_short8 a0;\n\
    vxc_half8 c0;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    src.hi = max(in, 0);\n\
    src.lo = min(in, 0);\n\
    VXC_DP2x8_b(dst, src.hi, src.lo, c0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluDFPLo_2x8b);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
#else\n\
_viv_uniform VXC_512Bits uniPreluInt8_2x8;\n\
_viv_uniform VXC_512Bits uniPreluInt16_part0_4x4;\n\
_viv_uniform VXC_512Bits uniPreluInt16_part1_4x4;\n\
__kernel void prelu_I8F16toI8_2D_OPT\n\
(\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 in, dst;\n\
    vxc_char16 src0, src1, src;\n\
    vxc_short8 a0, a1;\n\
    vxc_half8  c0, c1;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a1, param, coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    _viv_asm(COPY, c1, a1, 4);\n\
    src0 = max(in, 0);\n\
    src1 = min(in, 0);\n\
    _viv_asm(COPY, src, src0, 16);\n\
    src.s89abcdef = src1.s01234567;\n\
    VXC_DP2x8(dst, src, c0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluInt8_2x8);\n\
    _viv_asm(COPY, src, src1, 16);\n\
    src.s01234567 = src0.s89abcdef;\n\
    VXC_DP2x8(dst, src, c1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniPreluInt8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void prelu_I16F16toI16_2D_OPT\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t param,\n\
    image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_short8 in, dst;\n\
    vxc_short8 src0, src1, src;\n\
    vxc_short8 a0;\n\
    vxc_half8  c0;\n\
    VXC_ReadImage(in, input, coord.xy, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(a0, param, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, c0, a0, 4);\n\
    src0 = max(in, 0);\n\
    src1 = min(in, 0);\n\
    _viv_asm(COPY, src, src0, 16);\n\
    src.s4567 = src1.s0123;\n\
    VXC_DP4x4(dst, src, c0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniPreluInt16_part0_4x4);\n\
    _viv_asm(COPY, src, src1, 16);\n\
    src.s0123 = src0.s4567;\n\
    VXC_DP4x4(dst, src, c0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1), uniPreluInt16_part1_4x4);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
#endif\n\
\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDataSubZPtoFp32Part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_part0_4x4;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bin_2x8;\n\
_viv_uniform int inputZP0;\n\
_viv_uniform int inputZP1;\n\
_viv_uniform float input_scale0;\n\
_viv_uniform float input_scale1;\n\
_viv_uniform float outputZP;\n\
#define PRELU_F16_3D(name0, name1, input_type0, copy_type0, output_type, convert_type, copy_type) \\\n\
    __kernel void prelu_##name0##to##name1( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\\\n\
    vxc_float4 vecA, vecB, vecC, vecD;\\\n\
    input_type0 srcA;\\\n\
    copy_type0  src0;\\\n\
    vxc_short8 srcB;\\\n\
    vxc_half8  src1;\\\n\
    input_type0 input_ZP;\\\n\
    VXC_ReadImage2DArray(srcA, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src0, srcA, 16); \\\n\
    VXC_ReadImage2DArray(srcB, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src1, srcB, 16); \\\n\
    \\\n\
    _viv_asm(COPY, input_ZP, inputZP0, 4);\\\n\
    VXC_DP4x4(vecA, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), \\\n\
        uniDataSubZPtoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), \\\n\
        uniDataSubZPtoFp32Part1_4x4);\\\n\
    VXC_DP4x4(vecC, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part0_4x4);\\\n\
    VXC_DP4x4(vecD, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part1_4x4);\\\n\
    \\\n\
    vecA = vecA * input_scale0;\\\n\
    vecB = vecB * input_scale0;\\\n\
    vxc_float4 maxData0 = vecA > 0 ? vecA : 0.0; \\\n\
    vxc_float4 maxData1 = vecB > 0 ? vecB : 0.0; \\\n\
    vxc_float4 minData0 = vecA < 0 ? vecA : 0.0; \\\n\
    vxc_float4 minData1 = vecB < 0 ? vecB : 0.0; \\\n\
    vecA = maxData0 + vecC * minData0 + outputZP;\\\n\
    vecB = maxData1 + vecD * minData1 + outputZP;\\\n\
    convert_type dst0, dst1;\\\n\
    _viv_asm(CONV_RTE, dst0, vecA);\\\n\
    _viv_asm(CONV_RTE, dst1, vecB);\\\n\
    output_type dst2;\\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
    copy_type dst;\\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
}\n\
//        name0, name1, input_type0, copy_type0,  output_type, convert_type, copy_type\n\
PRELU_F16_3D(I8F16,  I8,  vxc_char16,  vxc_char16,  vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_3D(I8F16,  F16, vxc_char16,  vxc_char16,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(I16F16, I16, vxc_short8,  vxc_short8,  vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_3D(I16F16, F16, vxc_short8,  vxc_short8,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(U8F16,  U8,  vxc_uchar16, vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16)\n\
PRELU_F16_3D(U8F16,  F16, vxc_uchar16, vxc_uchar16, vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(F16F16, F16, vxc_short8,  vxc_half8,   vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_3D(F16F16, I8,  vxc_short8,  vxc_half8,   vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_3D(F16F16, I16, vxc_short8,  vxc_half8,   vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_3D(F16F16, U8,  vxc_short8,  vxc_half8,   vxc_uchar16, int4,  vxc_uchar16)\n\
\n\
#define PRELU_F16_2D(name0, name1, input_type0, copy_type0, output_type, convert_type, copy_type) \\\n\
    __kernel void prelu_##name0##to##name1##_2D( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    vxc_float4 vecA, vecB, vecC, vecD;\\\n\
    input_type0 srcA;\\\n\
    copy_type0  src0;\\\n\
    vxc_short8 srcB;\\\n\
    vxc_half8  src1;\\\n\
    input_type0 input_ZP;\\\n\
    VXC_ReadImage(srcA, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src0, srcA, 16); \\\n\
    VXC_ReadImage(srcB, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    _viv_asm(COPY, src1, srcB, 16); \\\n\
    \\\n\
    _viv_asm(COPY, input_ZP, inputZP0, 4);\\\n\
    VXC_DP4x4(vecA, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part0_4x4);\\\n\
    VXC_DP4x4(vecB, src0, input_ZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part1_4x4);\\\n\
    VXC_DP4x4(vecC, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part0_4x4);\\\n\
    VXC_DP4x4(vecD, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniConvF16toF32_part1_4x4);\\\n\
    \\\n\
    vecA = vecA * input_scale0;\\\n\
    vecB = vecB * input_scale0;\\\n\
    vxc_float4 maxData0 = vecA > 0 ? vecA : 0.0; \\\n\
    vxc_float4 maxData1 = vecB > 0 ? vecB : 0.0; \\\n\
    vxc_float4 minData0 = vecA < 0 ? vecA : 0.0; \\\n\
    vxc_float4 minData1 = vecB < 0 ? vecB : 0.0; \\\n\
    vecA = maxData0 + vecC * minData0 + outputZP;\\\n\
    vecB = maxData1 + vecD * minData1 + outputZP;\\\n\
    convert_type dst0, dst1;\\\n\
    _viv_asm(CONV_RTE, dst0, vecA);\\\n\
    _viv_asm(CONV_RTE, dst1, vecB);\\\n\
    output_type dst2;\\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
    copy_type dst;\\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
}\n\
PRELU_F16_2D(I8F16,  F16, vxc_char16,  vxc_char16,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(I8F16,  I8,  vxc_char16,  vxc_char16,  vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_2D(I16F16, F16, vxc_short8,  vxc_short8,  vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(U8F16,  U8,  vxc_uchar16, vxc_uchar16, vxc_uchar16, int4,  vxc_uchar16)\n\
PRELU_F16_2D(U8F16,  F16, vxc_uchar16, vxc_uchar16, vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(F16F16, F16, vxc_short8,  vxc_half8,   vxc_half8,   half4, vxc_short8)\n\
PRELU_F16_2D(F16F16, I8,  vxc_short8,  vxc_half8,   vxc_char16,  int4,  vxc_char16)\n\
PRELU_F16_2D(F16F16, I16, vxc_short8,  vxc_half8,   vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_2D(I16F16, I16, vxc_short8,  vxc_short8,  vxc_short8,  int4,  vxc_short8)\n\
PRELU_F16_2D(F16F16, U8,  vxc_short8,  vxc_half8,   vxc_uchar16, int4,  vxc_uchar16)\n\
\n\
#define PRELU_U8_2D(name, output_type, convert_type, copy_type) \\\n\
    __kernel void prelu_U8U8to##name##_2D( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output) \\\n\
{\\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\\\n\
    vxc_float4 vecA, vecB, vecC, vecD;\\\n\
    vxc_uchar16  src0;\\\n\
    vxc_uchar16  src1;\\\n\
    vxc_uchar16 input_ZP0;\\\n\
    vxc_uchar16 input_ZP1;\\\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
    \\\n\
    _viv_asm(COPY, input_ZP0, inputZP0, 4);\\\n\
    VXC_DP4x4(vecA, src0, input_ZP0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part0_4x4);\\\n\
    VXC_DP4x4(vecB, src0, input_ZP0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part1_4x4);\\\n\
    _viv_asm(COPY, input_ZP1, inputZP1, 4);\\\n\
    VXC_DP4x4(vecC, src1, input_ZP1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part0_4x4);\\\n\
    VXC_DP4x4(vecD, src1, input_ZP1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardInf, 0), uniDataSubZPtoFp32Part1_4x4);\\\n\
    \\\n\
    vecA = vecA * input_scale0;\\\n\
    vecB = vecB * input_scale0;\\\n\
    vecC = vecC * input_scale1;\\\n\
    vecD = vecD * input_scale1;\\\n\
    vxc_float4 maxData0 = vecA >= 0 ? vecA : 0.0; \\\n\
    vxc_float4 maxData1 = vecB >= 0 ? vecB : 0.0; \\\n\
    vxc_float4 minData0 = vecA < 0 ? vecA : 0.0; \\\n\
    vxc_float4 minData1 = vecB < 0 ? vecB : 0.0; \\\n\
    vecA = maxData0 + vecC * minData0 + outputZP;\\\n\
    vecB = maxData1 + vecD * minData1 + outputZP;\\\n\
    convert_type dst0, dst1;\\\n\
    _viv_asm(CONV_RTE, dst0, vecA);\\\n\
    _viv_asm(CONV_RTE, dst1, vecB);\\\n\
    output_type dst2;\\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bin_2x8);\\\n\
    copy_type dst;\\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\\\n\
}\n\
PRELU_U8_2D(U8,  vxc_uchar16, int4,  vxc_uchar16)\n\
PRELU_U8_2D(F16, vxc_half8,   half4, vxc_short8)\n\
\n\
\n\
"; /* end of prelu_vx*/

static const char prelu_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_Part0_4x4;\n\
_viv_uniform VXC_512Bits uniConvF16toF32_Part1_4x4;\n\
_viv_uniform VXC_512Bits uniPackedBF16_2x8;\n\
\n\
#define PRELU_BF16F16TOBF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, para_s16; \\\n\
    vxc_half8 para_f16; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(para_s16, param, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, para_f16, para_s16, 16); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_ushort8 src1, src2; \\\n\
    float4 srcA, srcB; \\\n\
    float4 para0_f32, para1_f32; \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, srcA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
    VXC_DP4x4(para0_f32, para_f16, para_f16, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvF16toF32_Part0_4x4);\\\n\
    VXC_DP4x4(para1_f32, para_f16, para_f16, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvF16toF32_Part1_4x4);\\\n\
    srcA = srcA >= 0 ? srcA : srcA * para0_f32; \\\n\
    srcB = srcB >= 0 ? srcB : srcB * para1_f32; \\\n\
    _viv_asm(COPY, src1, srcA, 16); \\\n\
    _viv_asm(COPY, src2, srcB, 16); \\\n\
    VXC_DP2x8(src1, src1, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackedBF16_2x8); \\\n\
    write_fun(output, coord, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void prelu_BF16F16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    PRELU_BF16F16TOBF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
__kernel void prelu_BF16F16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output,\n\
    int                          axis\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    PRELU_BF16F16TOBF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
#define PRELU_BF16BF16TOBF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 src0, para_s16; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(para_s16, param, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    vxc_ushort8 src1, src2; \\\n\
    float4 srcA, srcB; \\\n\
    float4 para0_f32, para1_f32; \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, srcA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
    VXC_DP2x8(src1, para_s16, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, para0_f32, src1, 16); \\\n\
    VXC_DP2x8(src1, para_s16, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, para1_f32, src1, 16); \\\n\
    srcA = srcA >= 0 ? srcA : srcA * para0_f32; \\\n\
    srcB = srcB >= 0 ? srcB : srcB * para1_f32; \\\n\
    _viv_asm(COPY, src1, srcA, 16); \\\n\
    _viv_asm(COPY, src2, srcB, 16); \\\n\
    VXC_DP2x8(src1, src1, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackedBF16_2x8); \\\n\
    write_fun(output, coord, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void prelu_BF16BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    PRELU_BF16BF16TOBF16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
__kernel void prelu_BF16BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_array_t param,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    PRELU_BF16BF16TOBF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
"; /* end of prelu_BF16_vx*/

static const char random_multinomial_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int class_max_iter;\n\
\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
_viv_uniform VXC_512Bits uniGetSubData0to3_4x4;\n\
_viv_uniform VXC_512Bits uniGetSubData4to7_4x4;\n\
_viv_uniform int iter;\n\
_viv_uniform int stride;\n\
_viv_uniform float re_rand_max;\n\
\n\
uint4 _philox4x32bumpkey(uint4 key)\n\
{\n\
    uint4 mask = (uint4)((uint)0x9E3779B9, (uint)0xBB67AE85, 0, 0);\n\
    //key.x += ((uint)0x9E3779B9);\n\
    //key.y += ((uint)0xBB67AE85);\n\
    key += mask;\n\
    return key;\n\
}\n\
\n\
uint mulhilo32(uint a, uint b, uint* hip)\n\
{\n\
    uint product = (uint)(a * b);\n\
    *hip = mul_hi(a, b);\n\
    return product;\n\
}\n\
\n\
uint mullo32(uint a, uint b)\n\
{\n\
    return a * b;\n\
}\n\
\n\
uint mulhi32(uint a, uint b)\n\
{\n\
    return mul_hi(a, b);\n\
}\n\
\n\
uint4 _philox4x32round(uint4 ctr, uint4 key)\n\
{\n\
    //uint hi0;\n\
    //uint hi1;\n\
    uint PHILOX_M4x32_0 = ((uint)0xD2511F53);\n\
    uint PHILOX_M4x32_1 = ((uint)0xCD9E8D57);\n\
    uint lo0 = mullo32(PHILOX_M4x32_0, ctr.x);\n\
    uint hi0 = mulhi32(PHILOX_M4x32_0, ctr.x);\n\
    uint lo1 = mullo32(PHILOX_M4x32_1, ctr.z);\n\
    uint hi1 = mulhi32(PHILOX_M4x32_1, ctr.z);\n\
    //uint lo0 = mulhilo32(PHILOX_M4x32_0, ctr.x, &hi0);\n\
    //uint lo1 = mulhilo32(PHILOX_M4x32_1, ctr.z, &hi1);\n\
    uint4 out = (uint4)(hi1^ctr.y^key.x, lo1, hi0^ctr.w^key.y, lo0);\n\
    return out;\n\
}\n\
\n\
uint4 philox4x32_R_10(uint4 ctr, uint4 key)\n\
{\n\
    uint i;\n\
    ctr = _philox4x32round(ctr, key);\n\
    for (i = 1; i < 10; i++)\n\
    {\n\
        key = _philox4x32bumpkey(key);\n\
        ctr = _philox4x32round(ctr, key);\n\
    }\n\
    return ctr;\n\
}\n\
\n\
__kernel void random_seed(\n\
    __read_only  image2d_t  seeds,\n\
    __write_only image2d_t  output)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx << 1, gidy, 0, 0);\n\
\n\
    int width = get_image_width(seeds);\n\
    Image s_img = create_image_from_image2d(seeds, 4);\n\
    __global uint* seeds_ptr = (__global uint*)s_img.ptr;\n\
    seeds_ptr = seeds_ptr + coord.x + coord.y * width;\n\
    uint4 key = vload4(0, seeds_ptr);\n\
\n\
    uint4 ctr = (uint4)(0);\n\
    float4 result = 0;\n\
\n\
    width = get_image_width(output);\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    coord.x = gidx * stride + width * coord.y;\n\
    __global float* output_ptr = (__global float*)o_img.ptr;\n\
    output_ptr += coord.x;\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        ctr = philox4x32_R_10(ctr, key);\n\
        result = convert_float4(ctr) * re_rand_max;\n\
        vstore4(result, i, output_ptr);\n\
    }\n\
}\n\
\n\
#define logE    (1.44269502f)\n\
float4 eltwise_unary_exp(float4 x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
// N times of 8\n\
// x dim = 1\n\
__kernel void random_multinomial_cdf_F16\n\
    (\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
\n\
    vxc_half8 maxData, data;\n\
    vxc_short8 src0;\n\
    float4 dst0 = 0, dst1 = 0;\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float tmp = 0;\n\
\n\
    int class_max_stride = get_image_width(input);\n\
    int offset = gidy * class_max_stride;\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    __global float* output_ptr = (__global float*)o_img.ptr;\n\
    __global float* cdfPtr = output_ptr + offset;\n\
\n\
    VXC_ReadImage(maxData, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.x += 8;\n\
    for(int i = 1; i < class_max_iter; i++)\n\
    {\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
        _viv_asm(COPY, data, src0, 16);\n\
\n\
        VXC_VertMax3_Half(maxData, maxData, maxData, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    VXC_HorzMax3_Half(maxData, maxData, VXC_MODIFIER(0, 5, 0,VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(maxData, maxData, maxData, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8);\n\
    VXC_HorzMax3_Half(maxData, maxData, VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord.x = 0;\n\
    for(int i = 0; i < class_max_iter; i++)\n\
    {\n\
        float4 val0, val1;\n\
        VXC_ReadImage(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
        _viv_asm(COPY, data, src0, 16);\n\
        VXC_DP4x4(val0, data, maxData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData0to3_4x4);\n\
        VXC_DP4x4(val1, data, maxData, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetSubData4to7_4x4);\n\
        val0 = eltwise_unary_exp(val0);\n\
        val1 = eltwise_unary_exp(val1);\n\
        val0.x += dst1.w;\n\
        dst0 = (float4)(val0.x, (val0.x + val0.y), dot(val0, (float4)(1, 1, 1, 0)), dot(val0, one));\n\
        val1.x += dst0.w;\n\
        dst0 = (float4)(val1.x, (val1.x + val1.y), dot(val1, (float4)(1, 1, 1, 0)), dot(val1, one));\n\
        vstore4(dst0, 0, cdfPtr);\n\
        vstore4(dst1, 1, cdfPtr);\n\
        cdfPtr += 8;\n\
    }\n\
}\n\
\n\
__kernel void random_multinomial_cdf_F32\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
\n\
    vxc_float4 src0, data;\n\
    float maxData0 = FLT_MIN, maxData1 = FLT_MIN;\n\
    uint4 ctr = (uint4)(0);\n\
    float4 dst = 0;\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
\n\
    int class_max_stride = get_image_width(input);\n\
    float tmp = 0;\n\
    int offset = gidy * class_max_stride;\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    __global float* output_ptr = (__global float*)o_img.ptr;\n\
    __global float* cdfPtr = output_ptr + offset;\n\
\n\
    int width = get_image_width(input);\n\
    Image i_img = create_image_from_image2d(input, 4);\n\
    __global float* input_ptr = (__global float*)i_img.ptr;\n\
    input_ptr = input_ptr + coord.x + coord.y * width;\n\
\n\
    float4 maxVal = vload4(0, input_ptr);\n\
    for(int i = 1; i < class_max_iter; i++)\n\
    {\n\
        src0 = vload4(i, input_ptr);\n\
\n\
        maxVal = maxVal > src0 ? maxVal : src0;\n\
    }\n\
    maxVal.xy = maxVal.xy > maxVal.zw ? maxVal.xy : maxVal.zw;\n\
    maxData0 = maxVal.x > maxVal.y ? maxVal.x : maxVal.y;\n\
\n\
    float4 maxData = (float4)(maxData0, maxData0, maxData0, maxData0);\n\
    for(int i = 0; i < class_max_iter; i++)\n\
    {\n\
        float4 val;\n\
        src0 = vload4(i, input_ptr);\n\
        data = src0 - maxData;\n\
        val = eltwise_unary_exp(data);\n\
        val.x += dst.w;\n\
        dst = (float4)(val.x, (val.x + val.y), dot(val, (float4)(1, 1, 1, 0)), dot(val, one));\n\
        vstore4(dst, i, cdfPtr);\n\
    }\n\
}\n\
\n\
uint upper_bound(float* a, int n, float x)\n\
{\n\
    uint l = 0;\n\
    uint h = n;\n\
    while (l < h) {\n\
        int mid = (l + h) >> 1;\n\
        if (x >= a[mid]) {\n\
            l = mid + 1;\n\
        } else {\n\
            h = mid;\n\
        }\n\
    }\n\
    return l;\n\
}\n\
\n\
// one thread calculate 4\n\
__kernel void random_multinomial\n\
    (\n\
    __read_only image2d_t   randoms,\n\
    __read_only image2d_t   cdfs,\n\
   __write_only image2d_t   output,\n\
                int         class_size\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
\n\
    int class_max_stride = get_image_width(cdfs);\n\
    int offset = gidy * class_max_stride;\n\
    Image cdf_img = create_image_from_image2d(cdfs, 4);\n\
    __global float* cdf_ptr = (__global float*)cdf_img.ptr;\n\
    __global float* cdfPtr = cdf_ptr + offset;\n\
\n\
    int width = get_image_width(randoms);\n\
    offset = coord.x + coord.y * width;\n\
    Image r_img = create_image_from_image2d(randoms, 4);\n\
    __global float* randoms_ptr = (__global float*)r_img.ptr;\n\
    randoms_ptr = randoms_ptr + offset;\n\
\n\
    width = get_image_width(output);\n\
    offset = coord.x + coord.y * width;\n\
    Image o_img = create_image_from_image2d(output, 4);\n\
    __global uint* output_ptr = (__global uint*)o_img.ptr;\n\
    output_ptr = output_ptr + offset;\n\
\n\
    float4 ran = vload4(0, randoms_ptr);\n\
    float total = cdfPtr[class_size - 1];\n\
    float4 target = ran * total;\n\
\n\
    uint4 out_class = (uint4)(0);\n\
    out_class.x = upper_bound(cdfPtr, class_size, target.x);\n\
    out_class.y = upper_bound(cdfPtr, class_size, target.y);\n\
    out_class.z = upper_bound(cdfPtr, class_size, target.z);\n\
    out_class.w = upper_bound(cdfPtr, class_size, target.w);\n\
\n\
    vstore4(out_class, 0, output_ptr);\n\
}\n\
\n\
"; /* end of random_multinomial_vx*/

static const char reduceall_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
\n\
_viv_uniform VXC_512Bits  uniS8AddAll_16x1;\n\
\n\
#define REDUCEALL_AXIS0_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    int4 sum_val = 0; \\\n\
    result = ones; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        VXC_DP16x1(sum_val, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniS8AddAll_16x1); \\\n\
        if (sum_val.x != 16) \\\n\
        { \\\n\
            result = zeros; \\\n\
            break; \\\n\
        } \\\n\
        coord.x += 16; \\\n\
    } \\\n\
    while(coord.x < axisSize); \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceall_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS0_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceall_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(0));\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS0_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceall_internal_axis0_vx*/

static const char reduceall_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEALL_AXIS1_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.y++; \\\n\
    while(coord.y < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result & val; \\\n\
        coord.y++; \\\n\
    } \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceall_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS1_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceall_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS1_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of reduceall_internal_axis1_vx*/

static const char reduceall_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEALL_AXIS2_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.z++; \\\n\
    while(coord.z < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result & val; \\\n\
        coord.z++; \\\n\
    } \\\n\
    write_fun(output, coord.xy, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceall_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEALL_AXIS2_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceall_internal_axis2_vx*/

static const char reduceany_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
\n\
_viv_uniform VXC_512Bits  uniS8AddAll_16x1;\n\
\n\
#define REDUCEANY_AXIS0_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    int4 sum_val = 0; \\\n\
    result = zeros; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        VXC_DP16x1(sum_val, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniS8AddAll_16x1); \\\n\
        if (sum_val.x != 0) \\\n\
        { \\\n\
            result = ones; \\\n\
            break; \\\n\
        } \\\n\
        coord.x += 16; \\\n\
    } \\\n\
    while(coord.x < axisSize); \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceany_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS0_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceany_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(0));\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS0_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceany_internal_axis0_vx*/

static const char reduceany_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEANY_AXIS1_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.y++; \\\n\
    while(coord.y < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result | val; \\\n\
        coord.y++; \\\n\
    } \\\n\
    write_fun(output, coord_out, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceany_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS1_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
__kernel void reduceany_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS1_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
"; /* end of reduceany_internal_axis1_vx*/

static const char reduceany_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
\n\
#define REDUCEANY_AXIS2_PROCESS(read_fun, write_fun) \\\n\
    vxc_char16 ones  = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; \\\n\
    vxc_char16 zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    result = val0 != zeros ? ones : zeros; \\\n\
    coord.z++; \\\n\
    while(coord.z < axisSize) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        val = val0 != zeros ? ones : zeros; \\\n\
        result = result | val; \\\n\
        coord.z++; \\\n\
    } \\\n\
    write_fun(output, coord.xy, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void reduceany_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_char16 val0;\n\
    vxc_char16 val, result;\n\
    REDUCEANY_AXIS2_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage)\n\
}\n\
\n\
\n\
"; /* end of reduceany_internal_axis2_vx*/

static const char reducemax_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
\n\
#define REDUCEMAX_PROCESS_AXIS0(read_fun, vert_max_fun, horz_max_fun) \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        vert_max_fun(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        vert_max_fun(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        horz_max_fun(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
        horz_max_fun(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS0_SAVE_SAME(save_type, write_fun) \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, val, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
                      OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    vxc_float4 prob; \\\n\
    dst_type vec1; \\\n\
    save_type dst; \\\n\
    VXC_DP4x4(prob, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    prob = ((prob - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, vec1, prob); \\\n\
    _viv_asm(COPY, dst, vec1, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_AXIS0_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
#define REDUCEMAX_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
              save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMAX_AXIS0_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
\n\
REDUCEMAX_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric,\\\n\
                    1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
REDUCEMAX_AXIS0(I16, I16, vxc_short8, vxc_short8, short4,\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale,\\\n\
input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
#define REDUCEMAX_AXIS0_SAME_2D(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode,\\\n\
                           OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun, horz_max_fun) \\\n\
__kernel void reducemax_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMAX_PROCESS_AXIS0(VXC_ReadImage, vert_max_fun, horz_max_fun); \\\n\
    REDUCEMAX_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMAX_AXIS0_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
\n\
REDUCEMAX_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric,\\\n\
                    1, 0, VXC_VertMax3_Half, VXC_HorzMax3_Half)\n\
REDUCEMAX_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,\n\
                   vxc_short8, CONV, 1, 0, inputScale,\\\n\
                   input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\n\
                   CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\n\
                   CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
REDUCEMAX_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale,\\\n\
input_offset_asymmetric, VXC_VertMax3_Integer, VXC_HorzMax3_Integer)\n\
\n\
\n\
"; /* end of reducemax_internal_axis0_vx*/

static const char reducemax_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMAX_PROCESS_AXIS1(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define REDUCEMAX_PROCESS_AXIS1_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, max, 16); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_AXIS1_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS1(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMAX_AXIS1_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half)\n\
\n\
\n\
REDUCEMAX_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
\n\
\n\
#define REDUCEMAX_AXIS1_SAME_2D(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS1_2D(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun) \\\n\
__kernel void reducemax_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS1(VXC_ReadImage, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMAX_AXIS1_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half)\n\
\n\
REDUCEMAX_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,\n\
vxc_short8, CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(I16, I16, vxc_short8, vxc_short8,\\\n\
short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16,\\\n\
char4, vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
"; /* end of reducemax_internal_axis1_vx*/

static const char reducemax_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMAX_PROCESS_AXIS2(read_fun, vert_max_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, max, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_max_fun(max, max, max, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
#define REDUCEMAX_PROCESS_AXIS2_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, max, 16); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, max, max, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMAX_AXIS2_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_max_fun) \\\n\
__kernel void reducemax_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS2_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMAX_AXIS2(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_max_fun) \\\n\
__kernel void reducemax_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, max; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMAX_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_max_fun) \\\n\
    REDUCEMAX_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode, OUT_SCALE,\\\n\
    OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMAX_AXIS2_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMax3_Half)\n\
\n\
\n\
REDUCEMAX_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8, \\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMax3_Half)\n\
REDUCEMAX_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
\n\
REDUCEMAX_AXIS2(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMax3_Integer)\n\
REDUCEMAX_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMax3_Integer)\n\
\n\
"; /* end of reducemax_internal_axis2_vx*/

static const char reducemin_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniPackMaxData_2x8;\n\
\n\
#define REDUCEMIN_PROCESS_AXIS0(read_fun, vert_min_fun, horz_min_fun) \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, val, val0, 16); \\\n\
    coord.x += 8; \\\n\
    do \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(-16, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val0, val0, 16); \\\n\
        read_fun(val1, input,  coord, VXC_5BITOFFSET_XY(-8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val1, val1, 16); \\\n\
        read_fun(val2, input,  coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val2, val2, 16); \\\n\
        read_fun(val3, input,  coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val3, val3, 16); \\\n\
        coord.x += 32; \\\n\
        vert_min_fun(val, img_val0, img_val1, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        vert_min_fun(val, img_val2, img_val3, val, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    while(coord.x < (axisSize + 16)); \\\n\
    horz_min_fun(val, val, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val, val, val, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0), uniPackMaxData_2x8); \\\n\
    horz_min_fun(val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS0_SAVE_SAME(save_type, write_fun) \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, val, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    vxc_float4 prob; \\\n\
    dst_type vec1; \\\n\
    save_type dst; \\\n\
    VXC_DP4x4(prob, val, val, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    prob = ((prob - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, vec1, prob); \\\n\
    _viv_asm(COPY, dst, vec1, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_AXIS0_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
#define REDUCEMIN_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
              save_type, conv_mode, OUT_SCALE, OUT_OFFSET,\\\n\
              IN_SCALE, IN_OFFSET, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(16, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage2DArray, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMIN_AXIS0_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
\n\
REDUCEMIN_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale,\\\n\
0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale,\\\n\
0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
 CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0,\\\n\
inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0,\\\n\
inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8, CONV,\\\n\
1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
\n\
REDUCEMIN_AXIS0(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric,\\\n\
VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
\n\
#define REDUCEMIN_AXIS0_SAME_2D(src_name, dst_name, src_type, copy_type,\\\n\
save_type, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode, OUT_SCALE,\\\n\
                           OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun, horz_min_fun) \\\n\
__kernel void reducemin_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(16, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type img_val0, img_val1, img_val2, img_val3; \\\n\
    copy_type val0, val1, val2, val3; \\\n\
    src_type val; \\\n\
    REDUCEMIN_PROCESS_AXIS0(VXC_ReadImage, vert_min_fun, horz_min_fun); \\\n\
    REDUCEMIN_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMIN_AXIS0_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
\n\
REDUCEMIN_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
REDUCEMIN_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half, VXC_HorzMin3_Half)\n\
\n\
REDUCEMIN_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
REDUCEMIN_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric,\\\n\
VXC_VertMin3_Integer, VXC_HorzMin3_Integer)\n\
\n\
\n\
"; /* end of reducemin_internal_axis0_vx*/

static const char reducemin_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMIN_PROCESS_AXIS1(read_fun, vert_min_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, min, in0, 16); \\\n\
    coord.y++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_min_fun(min, min, min, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define REDUCEMIN_PROCESS_AXIS1_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, min, 16); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_AXIS1_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS1(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMIN_AXIS1_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half)\n\
\n\
\n\
REDUCEMIN_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
\n\
\n\
#define REDUCEMIN_AXIS1_SAME_2D(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS1_2D(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun) \\\n\
__kernel void reducemin_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS1(VXC_ReadImage, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEMIN_AXIS1_SAME_2D(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half)\n\
\n\
REDUCEMIN_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
"; /* end of reducemin_internal_axis1_vx*/

static const char reducemin_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
\n\
#define REDUCEMIN_PROCESS_AXIS2(read_fun, vert_min_fun) \\\n\
    read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, min, in0, 16); \\\n\
    coord.z++; \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        vert_min_fun(min, min, min, vec0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
#define REDUCEMIN_PROCESS_AXIS2_SAVE_SAME(save_type, write_fun) \\\n\
    save_type vect; \\\n\
    _viv_asm(COPY, vect, min, 16); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, data0); \\\n\
    VXC_DP4x4(data0, min, min, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    data0 = ((data0 - IN_OFFSET) * IN_SCALE) * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, data0); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEMIN_AXIS2_SAME(src_name, dst_name, src_type, copy_type, save_type, vert_min_fun) \\\n\
__kernel void reducemin_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS2_SAVE_SAME(save_type, VXC_WriteImage); \\\n\
}\n\
\n\
#define REDUCEMIN_AXIS2(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, vert_min_fun) \\\n\
__kernel void reducemin_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    src_type vec0, min; \\\n\
    copy_type in0; \\\n\
    vxc_float4 data0; \\\n\
    REDUCEMIN_PROCESS_AXIS2(VXC_ReadImage2DArray, vert_min_fun) \\\n\
    REDUCEMIN_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEMIN_AXIS2_SAME(F16, F16, vxc_half8, vxc_short8,  vxc_short8, VXC_VertMin3_Half)\n\
\n\
\n\
REDUCEMIN_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0, VXC_VertMin3_Half)\n\
REDUCEMIN_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
\n\
REDUCEMIN_AXIS2(I16, I16, vxc_short8, vxc_short8, short4, vxc_short8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(I8, I8,  vxc_char16, vxc_char16,  char4, vxc_char8,\\\n\
CONV_SAT_RTE, outputScale, 0, inputScale, 0, VXC_VertMin3_Integer)\n\
REDUCEMIN_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric, VXC_VertMin3_Integer)\n\
\n\
"; /* end of reducemin_internal_axis2_vx*/

static const char reduceprod_internal_axis0_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
_viv_uniform int         inputWidth;\n\
_viv_uniform VXC_512Bits uniGetEndLoData_2x8;\n\
_viv_uniform VXC_512Bits uniGetEndHiData_2x8;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0(read_fun, IN_SCALE, IN_OFFSET) \\\n\
    while(coord.x < inputWidth) \\\n\
    { \\\n\
        read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, img_val, val0, 16); \\\n\
        VXC_DP4x4(tmpProdLo, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
        VXC_DP4x4(tmpProdHi, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
        tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProd = tmpProdLo * tmpProdHi; \\\n\
        prodValue = prodValue * tmpProd; \\\n\
        coord.x += 8; \\\n\
    } \\\n\
    vxc_ushort8 tmpProdInt0, tmpProdInt1; \\\n\
    vxc_ushort8 tmpOnesInt = {0, 16256, 0, 16256, 0, 16256, 0, 16256}; \\\n\
    read_fun(val0, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, img_val, val0, 16); \\\n\
    VXC_DP4x4(tmpProdLo, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
    VXC_DP4x4(tmpProdHi, img_val, img_val, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
    tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
    tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
    _viv_asm(COPY, tmpProdInt0, tmpProdLo, 16); \\\n\
    _viv_asm(COPY, tmpProdInt1, tmpProdHi, 16); \\\n\
    VXC_DP2x8(tmpProdInt0, tmpProdInt0, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndLoData_2x8); \\\n\
    VXC_DP2x8(tmpProdInt1, tmpProdInt1, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndHiData_2x8); \\\n\
    _viv_asm(COPY, tmpProdLo, tmpProdInt0, 16); \\\n\
    _viv_asm(COPY, tmpProdHi, tmpProdInt1, 16); \\\n\
    tmpProd = tmpProdLo * tmpProdHi; \\\n\
    prodValue = prodValue * tmpProd; \\\n\
    tmpProd.xy = prodValue.xy * prodValue.zw; \\\n\
    prodValue.x = tmpProd.x * tmpProd.y;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, write_fun) \\\n\
    dst_type vec1; \\\n\
    save_type dst; \\\n\
    prodValue = prodValue * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, vec1, prodValue); \\\n\
    _viv_asm(COPY, dst, vec1, 16); \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define REDUCEPROD_AXIS0(src_name, dst_name, src_type, copy_type, dst_type,\\\n\
              save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis0_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd; \\\n\
    src_type img_val; \\\n\
    copy_type val0; \\\n\
    REDUCEPROD_PROCESS_AXIS0(VXC_ReadImage2DArray, IN_SCALE, IN_OFFSET); \\\n\
    REDUCEPROD_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEPROD_AXIS0(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS0(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS0(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
\n\
REDUCEPROD_AXIS0(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
#define REDUCEPROD_AXIS0_2D(src_name, dst_name, src_type, copy_type,\\\n\
                           dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis0_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(0, get_global_id(0)); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd; \\\n\
    src_type img_val; \\\n\
    copy_type val0; \\\n\
    REDUCEPROD_PROCESS_AXIS0(VXC_ReadImage, IN_SCALE, IN_OFFSET); \\\n\
    REDUCEPROD_PROCESS_AXIS0_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEPROD_AXIS0_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS0_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS0_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
                    CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS0_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4, \\\n\
vxc_short8, CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
REDUCEPROD_AXIS0_2D(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS0_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0_BF16(read_fun) \\\n\
    while(coord.x < inputWidth) \\\n\
    { \\\n\
        read_fun(img_val, input,  coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, tmpProdLo, val0, 16); \\\n\
        VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, tmpProdHi, val0, 16); \\\n\
        tmpProd = tmpProdLo * tmpProdHi; \\\n\
        prodValue = prodValue * tmpProd; \\\n\
        coord.x += 8; \\\n\
    } \\\n\
    vxc_ushort8 tmpProdInt0, tmpProdInt1; \\\n\
    vxc_ushort8 tmpOnesInt = {0, 16256, 0, 16256, 0, 16256, 0, 16256}; \\\n\
    read_fun(img_val, input,  coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, tmpProdLo, val0, 16); \\\n\
    VXC_DP2x8(val0, img_val, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, tmpProdHi, val0, 16); \\\n\
    _viv_asm(COPY, tmpProdInt0, tmpProdLo, 16); \\\n\
    _viv_asm(COPY, tmpProdInt1, tmpProdHi, 16); \\\n\
    VXC_DP2x8(tmpProdInt0, tmpProdInt0, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndLoData_2x8); \\\n\
    VXC_DP2x8(tmpProdInt1, tmpProdInt1, tmpOnesInt,\\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetEndHiData_2x8); \\\n\
    _viv_asm(COPY, tmpProdLo, tmpProdInt0, 16); \\\n\
    _viv_asm(COPY, tmpProdHi, tmpProdInt1, 16); \\\n\
    tmpProd = tmpProdLo * tmpProdHi; \\\n\
    prodValue = prodValue * tmpProd; \\\n\
    tmpProd.xy = prodValue.xy * prodValue.zw; \\\n\
    prodValue.x = tmpProd.x * tmpProd.y;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS0_BF16_SAVE(write_fun) \\\n\
    vxc_ushort8 dst; \\\n\
    _viv_asm(COPY, dst, prodValue, 16); \\\n\
    dst.s0 = dst.s1; \\\n\
    write_fun(output, coord_out, dst, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void reduceprod_axis0_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd;\n\
    vxc_short8 img_val;\n\
    vxc_short8 val0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16(VXC_ReadImage2DArray);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16_SAVE(VXC_WriteImage);\n\
}\n\
\n\
__kernel void reduceprod_axis0_BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(0));\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_float4 prodValue = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi, tmpProd;\n\
    vxc_short8 img_val;\n\
    vxc_short8 val0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16(VXC_ReadImage);\n\
    REDUCEPROD_PROCESS_AXIS0_BF16_SAVE(VXC_WriteImage);\n\
}\n\
\n\
"; /* end of reduceprod_internal_axis0_vx*/

static const char reduceprod_internal_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1(read_fun, IN_SCALE, IN_OFFSET) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(tmpProdLo, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
        VXC_DP4x4(tmpProdHi, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
        tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    prodValueLo = prodValueLo * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, prodValueLo); \\\n\
    prodValueHi = prodValueHi * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, prodValueHi); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define REDUCEPROD_AXIS1(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis1_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\\\n\
    vxc_float4 tmpProdLo, tmpProdHi; \\\n\
    src_type vec0; \\\n\
    copy_type in0; \\\n\
    REDUCEPROD_PROCESS_AXIS1(VXC_ReadImage2DArray, IN_SCALE, IN_OFFSET) \\\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEPROD_AXIS1(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS1(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS1(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(U8, F16, vxc_uchar16, vxc_uchar16, half4, \\\n\
vxc_short8, CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
REDUCEPROD_AXIS1(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
\n\
#define REDUCEPROD_AXIS1_2D(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis1_##src_name##to##dst_name##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    int2 coord_out = (int2)(get_global_id(0), 0); \\\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\\\n\
    vxc_float4 tmpProdLo, tmpProdHi; \\\n\
    src_type vec0; \\\n\
    copy_type in0; \\\n\
    REDUCEPROD_PROCESS_AXIS1(VXC_ReadImage, IN_SCALE, IN_OFFSET) \\\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, VXC_WriteImage); \\\n\
}\n\
\n\
REDUCEPROD_AXIS1_2D(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS1_2D(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1_2D(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS1_2D(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS1_2D(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
REDUCEPROD_AXIS1_2D(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS1_2D(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1_BF16(read_fun) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, tmpProdLo, vec0, 16); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, tmpProdHi, vec0, 16); \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.y++; \\\n\
    } \\\n\
    while(coord.y < axisSize);\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS1_SAVE_BF16(write_fun) \\\n\
    vxc_ushort8 dst0, dst1; \\\n\
    vxc_ushort8 vect; \\\n\
    _viv_asm(COPY, dst0, prodValueLo, 16); \\\n\
    _viv_asm(COPY, dst1, prodValueHi, 16); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord_out, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void reduceprod_axis1_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), get_global_id(1));\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi;\n\
    vxc_short8 vec0;\n\
    vxc_short8 in0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS1_BF16(VXC_ReadImage2DArray)\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE_BF16(VXC_WriteImage);\n\
}\n\
\n\
__kernel void reduceprod_axis1_BF16toBF16_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), 0);\n\
    int2 coord_out = (int2)(get_global_id(0), 0);\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi;\n\
    vxc_short8 vec0;\n\
    vxc_short8 in0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS1_BF16(VXC_ReadImage)\n\
    REDUCEPROD_PROCESS_AXIS1_SAVE_BF16(VXC_WriteImage);\n\
}\n\
"; /* end of reduceprod_internal_axis1_vx*/

static const char reduceprod_internal_axis2_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float       outputScale;\n\
_viv_uniform float       output_offset_asymmetric;\n\
_viv_uniform float       inputScale;\n\
_viv_uniform float       input_offset_asymmetric;\n\
_viv_uniform VXC_512Bits uniGetLoData_4x4;\n\
_viv_uniform VXC_512Bits uniGetHiData_4x4;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
\n\
_viv_uniform int         axisSize;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2(read_fun, IN_SCALE, IN_OFFSET) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, vec0, in0, 16); \\\n\
        VXC_DP4x4(tmpProdLo, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetLoData_4x4); \\\n\
        VXC_DP4x4(tmpProdHi, vec0, vec0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetHiData_4x4); \\\n\
        tmpProdLo = (tmpProdLo - IN_OFFSET) * IN_SCALE; \\\n\
        tmpProdHi = (tmpProdHi - IN_OFFSET) * IN_SCALE; \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode, OUT_SCALE, OUT_OFFSET, write_fun) \\\n\
    dst_type dst0, dst1; \\\n\
    save_type vect; \\\n\
    prodValueLo = prodValueLo * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst0, prodValueLo); \\\n\
    prodValueHi = prodValueHi * OUT_SCALE + OUT_OFFSET; \\\n\
    _viv_asm(conv_mode, dst1, prodValueHi); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertInt32toUint8_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define REDUCEPROD_AXIS2(src_name, dst_name, src_type, copy_type, dst_type, save_type,\\\n\
conv_mode, OUT_SCALE, OUT_OFFSET, IN_SCALE, IN_OFFSET) \\\n\
__kernel void reduceprod_axis2_##src_name##to##dst_name \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
    int   axisVal \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0); \\\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f}; \\\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\\\n\
    vxc_float4 tmpProdLo, tmpProdHi; \\\n\
    src_type vec0; \\\n\
    copy_type in0; \\\n\
    REDUCEPROD_PROCESS_AXIS2(VXC_ReadImage2DArray, IN_SCALE, IN_OFFSET) \\\n\
    REDUCEPROD_PROCESS_AXIS2_SAVE(dst_type, save_type, conv_mode,\\\n\
    OUT_SCALE, OUT_OFFSET,  VXC_WriteImage); \\\n\
}\n\
\n\
\n\
\n\
REDUCEPROD_AXIS2(F16, F16, vxc_half8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, 1, 0)\n\
REDUCEPROD_AXIS2(F16, I16, vxc_half8, vxc_short8, short4, vxc_short8, CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS2(F16, I8,  vxc_half8, vxc_short8, char4,  vxc_char8,  CONV_SAT_RTE, outputScale, 0, 1, 0)\n\
REDUCEPROD_AXIS2(F16, U8,  vxc_half8, vxc_short8, uchar4, vxc_uchar8,\\\n\
CONV_SAT_RTE, outputScale, output_offset_asymmetric, 1, 0)\n\
REDUCEPROD_AXIS2(I16, F16, vxc_short8, vxc_short8, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(I8, F16, vxc_char16, vxc_char16, half4,  vxc_short8, CONV, 1, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(U8, F16, vxc_uchar16, vxc_uchar16, half4,  vxc_short8,\\\n\
CONV, 1, 0, inputScale, input_offset_asymmetric)\n\
\n\
REDUCEPROD_AXIS2(I16, I16, vxc_short8, vxc_short8, short4,\\\n\
vxc_short8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(I8, I8,  vxc_char16, vxc_char16,  char4,\\\n\
vxc_char8, CONV_SAT_RTE, outputScale, 0, inputScale, 0)\n\
REDUCEPROD_AXIS2(U8, U8,  vxc_uchar16, vxc_uchar16, uchar4, vxc_uchar8, CONV_SAT_RTE,\\\n\
outputScale, output_offset_asymmetric, inputScale, input_offset_asymmetric)\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2_BF16(read_fun) \\\n\
    do \\\n\
    { \\\n\
        read_fun(in0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
        _viv_asm(COPY, tmpProdLo, vec0, 16); \\\n\
        VXC_DP2x8(vec0, in0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
        _viv_asm(COPY, tmpProdHi, vec0, 16); \\\n\
        prodValueLo = prodValueLo * tmpProdLo; \\\n\
        prodValueHi = prodValueHi * tmpProdHi; \\\n\
        coord.z++; \\\n\
    } \\\n\
    while(coord.z < axisSize);\n\
\n\
\n\
#define REDUCEPROD_PROCESS_AXIS2_SAVE_BF16(write_fun) \\\n\
    vxc_ushort8 dst0, dst1; \\\n\
    vxc_ushort8 vect; \\\n\
    _viv_asm(COPY, dst0, prodValueLo, 16); \\\n\
    _viv_asm(COPY, dst1, prodValueHi, 16); \\\n\
    VXC_DP2x8(vect, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord.xy, vect, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void reduceprod_axis2_BF16toBF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   axisVal\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    vxc_float4 prodValueLo = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 prodValueHi = {1.0f, 1.0f, 1.0f, 1.0f};\n\
    vxc_float4 tmpProdLo, tmpProdHi;\n\
    vxc_short8 vec0;\n\
    vxc_short8 in0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    REDUCEPROD_PROCESS_AXIS2_BF16(VXC_ReadImage2DArray)\n\
    REDUCEPROD_PROCESS_AXIS2_SAVE_BF16(VXC_WriteImage);\n\
}\n\
"; /* end of reduceprod_internal_axis2_vx*/

static const char relational_ops_2d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float input0Scale;\n\
_viv_uniform float input0Tail;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform float input1Tail;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define COMPARISONS_2D(func_name, src0_type_name, src1_type_name, \\\n\
        src0_type, src0_copy_type, src1_type, src1_copy_type, cmp_op) \\\n\
__kernel void func_name##_##src0_type_name##src1_type_name##toBOOL8_2D( \\\n\
    __read_only  image2d_array_t  input0, \\\n\
    __read_only  image2d_array_t  input1, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src0_type      src0; \\\n\
    src0_copy_type srcA; \\\n\
    src0_type      src1; \\\n\
    src0_copy_type srcB; \\\n\
    VXC_ReadImage(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcA, src0, 16); \\\n\
    VXC_ReadImage(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
 \\\n\
    float4 vecA0, vecA1; \\\n\
    float4 vecB0, vecB1; \\\n\
    VXC_DP4x4(vecA0, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecA1, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA0 = vecA0 * input0Scale + input0Tail; \\\n\
    vecA1 = vecA1 * input0Scale + input0Tail; \\\n\
    VXC_DP4x4(vecB0, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB1, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecB0 = vecB0 * input1Scale + input1Tail; \\\n\
    vecB1 = vecB1 * input1Scale + input1Tail; \\\n\
    int4 dst0, dst1; \\\n\
    dst0 = (vecA0)cmp_op(vecB0); \\\n\
    dst1 = (vecA1)cmp_op(vecB1); \\\n\
 \\\n\
    vxc_char16 dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst &= 1; \\\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//LESS\n\
COMPARISONS_2D(less, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_2D(less, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_2D(less, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_2D(less, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <)\n\
COMPARISONS_2D(less, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_2D(less, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_2D(less, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_2D(less, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <)\n\
COMPARISONS_2D(less, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <)\n\
COMPARISONS_2D(less, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <)\n\
//GREAT\n\
COMPARISONS_2D(great, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_2D(great, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_2D(great, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_2D(great, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >)\n\
COMPARISONS_2D(great, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_2D(great, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_2D(great, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_2D(great, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >)\n\
COMPARISONS_2D(great, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >)\n\
COMPARISONS_2D(great, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >)\n\
//LESS_EQUAL\n\
COMPARISONS_2D(less_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_2D(less_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_2D(less_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_2D(less_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <=)\n\
COMPARISONS_2D(less_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_2D(less_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_2D(less_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_2D(less_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_2D(less_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <=)\n\
COMPARISONS_2D(less_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <=)\n\
//GREAT_EQUAL\n\
COMPARISONS_2D(great_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_2D(great_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_2D(great_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_2D(great_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >=)\n\
COMPARISONS_2D(great_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_2D(great_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_2D(great_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_2D(great_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_2D(great_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >=)\n\
COMPARISONS_2D(great_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >=)\n\
//EQUAL\n\
COMPARISONS_2D(equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_2D(equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_2D(equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_2D(equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, ==)\n\
COMPARISONS_2D(equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_2D(equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_2D(equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_2D(equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_2D(equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, ==)\n\
COMPARISONS_2D(equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  ==)\n\
//NOT_EQUAL\n\
COMPARISONS_2D(not_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_2D(not_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_2D(not_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_2D(not_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, !=)\n\
COMPARISONS_2D(not_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_2D(not_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_2D(not_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_2D(not_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_2D(not_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, !=)\n\
COMPARISONS_2D(not_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  !=)\n\
\n\
"; /* end of relational_ops_2d_vx*/

static const char relational_ops_3d_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float input0Scale;\n\
_viv_uniform float input0Tail;\n\
_viv_uniform float input1Scale;\n\
_viv_uniform float input1Tail;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define COMPARISONS_3D(func_name, src0_type_name, src1_type_name, \\\n\
        src0_type, src0_copy_type, src1_type, src1_copy_type, cmp_op) \\\n\
__kernel void func_name##_##src0_type_name##src1_type_name##toBOOL8( \\\n\
    __read_only  image2d_array_t  input0, \\\n\
    __read_only  image2d_array_t  input1, \\\n\
    __write_only image2d_array_t  output \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src0_type      src0; \\\n\
    src0_copy_type srcA; \\\n\
    src0_type      src1; \\\n\
    src0_copy_type srcB; \\\n\
    VXC_ReadImage2DArray(src0, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcA, src0, 16); \\\n\
    VXC_ReadImage2DArray(src1, input1, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, srcB, src1, 16); \\\n\
 \\\n\
    float4 vecA0, vecA1, vecA; \\\n\
    float4 vecB0, vecB1, vecB; \\\n\
    VXC_DP4x4(vecA0, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecA1, srcA, srcA, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA0 = vecA0 * input0Scale + input0Tail; \\\n\
    vecA1 = vecA1 * input0Scale + input0Tail; \\\n\
    VXC_DP4x4(vecB0, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB1, srcB, srcB, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecB0 = vecB0 * input1Scale + input1Tail; \\\n\
    vecB1 = vecB1 * input1Scale + input1Tail; \\\n\
    int4 dst0, dst1; \\\n\
    dst0 = (vecA0)cmp_op(vecB0); \\\n\
    dst1 = (vecA1)cmp_op(vecB1); \\\n\
 \\\n\
    vxc_char16 dst; \\\n\
    VXC_DP2x8(dst, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst &= 1; \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
//LESS\n\
COMPARISONS_3D(less, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_3D(less, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_3D(less, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_3D(less, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <)\n\
COMPARISONS_3D(less, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <)\n\
COMPARISONS_3D(less, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <)\n\
COMPARISONS_3D(less, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <)\n\
COMPARISONS_3D(less, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <)\n\
COMPARISONS_3D(less, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <)\n\
COMPARISONS_3D(less, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <)\n\
//GREAT\n\
COMPARISONS_3D(great, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_3D(great, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_3D(great, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_3D(great, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >)\n\
COMPARISONS_3D(great, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >)\n\
COMPARISONS_3D(great, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >)\n\
COMPARISONS_3D(great, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >)\n\
COMPARISONS_3D(great, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >)\n\
COMPARISONS_3D(great, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >)\n\
COMPARISONS_3D(great, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >)\n\
//LESS_EQUAL\n\
COMPARISONS_3D(less_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_3D(less_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_3D(less_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_3D(less_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, <=)\n\
COMPARISONS_3D(less_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  <=)\n\
COMPARISONS_3D(less_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_3D(less_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, <=)\n\
COMPARISONS_3D(less_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  <=)\n\
COMPARISONS_3D(less_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, <=)\n\
COMPARISONS_3D(less_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  <=)\n\
//GREAT_EQUAL\n\
COMPARISONS_3D(great_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_3D(great_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_3D(great_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_3D(great_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, >=)\n\
COMPARISONS_3D(great_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  >=)\n\
COMPARISONS_3D(great_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_3D(great_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, >=)\n\
COMPARISONS_3D(great_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  >=)\n\
COMPARISONS_3D(great_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, >=)\n\
COMPARISONS_3D(great_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  >=)\n\
//EQUAL\n\
COMPARISONS_3D(equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_3D(equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_3D(equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_3D(equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, ==)\n\
COMPARISONS_3D(equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  ==)\n\
COMPARISONS_3D(equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_3D(equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, ==)\n\
COMPARISONS_3D(equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  ==)\n\
COMPARISONS_3D(equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, ==)\n\
COMPARISONS_3D(equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  ==)\n\
//NOT_EQUAL\n\
COMPARISONS_3D(not_equal, F16, F16, vxc_short8, vxc_half8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_3D(not_equal, F16, I8,  vxc_short8, vxc_half8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_3D(not_equal, F16, U8,  vxc_short8, vxc_half8,  vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_3D(not_equal, F16, I16, vxc_short8, vxc_half8,  vxc_short8, vxc_short8, !=)\n\
COMPARISONS_3D(not_equal, I8,  I8,  vxc_char8,  vxc_char8,  vxc_char8,  vxc_char8,  !=)\n\
COMPARISONS_3D(not_equal, I8,  F16, vxc_char8,  vxc_char8,  vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_3D(not_equal, U8,  U8,  vxc_uchar8, vxc_uchar8, vxc_uchar8, vxc_uchar8, !=)\n\
COMPARISONS_3D(not_equal, U8,  F16, vxc_uchar8, vxc_uchar8, vxc_short8, vxc_half8,  !=)\n\
COMPARISONS_3D(not_equal, I16, I16, vxc_short8, vxc_short8, vxc_short8, vxc_short8, !=)\n\
COMPARISONS_3D(not_equal, I16, F16, vxc_short8, vxc_short8, vxc_short8, vxc_half8,  !=)\n\
\n\
"; /* end of relational_ops_3d_vx*/

static const char relu_keras_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvFP16toFP32_Lo_4x4;\n\
_viv_uniform VXC_512Bits uniConvFP16toFP32_Hi_4x4;\n\
_viv_uniform VXC_512Bits uniExtractHalf8_2x8;\n\
_viv_uniform VXC_512Bits uniExtractInteger_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniPackedBF16_2x8;\n\
_viv_uniform VXC_512Bits uniConvIntegertoFP32_Lo_4x4;\n\
_viv_uniform VXC_512Bits uniConvIntegertoFP32_Hi_4x4;\n\
_viv_uniform float offset;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float outputZP;\n\
\n\
float4 I8toF32_Lo(vxc_char8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Lo_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 I8toF32_Hi(vxc_char8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Hi_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 U8toF32_Lo(vxc_uchar8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Lo_4x4);\n\
    dst = dst * input_scale + inputTail;\n\
    return dst;\n\
}\n\
\n\
float4 U8toF32_Hi(vxc_uchar8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Hi_4x4);\n\
    dst = dst * input_scale + inputTail;\n\
    return dst;\n\
}\n\
\n\
float4 I16toF32_Lo(vxc_short8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Lo_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 I16toF32_Hi(vxc_short8 src)\n\
{\n\
    float4 dst;\n\
\n\
    VXC_DP4x4(dst, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvIntegertoFP32_Hi_4x4);\n\
    dst *= input_scale;\n\
    return dst;\n\
}\n\
\n\
float4 F16toF32_Lo(vxc_ushort8 src)\n\
{\n\
    vxc_half8 srcHalf;\n\
    float4 dst;\n\
\n\
    _viv_asm(COPY, srcHalf, src, 16);\n\
    VXC_DP4x4(dst, srcHalf, srcHalf, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvFP16toFP32_Lo_4x4);\n\
    return dst;\n\
}\n\
\n\
float4 F16toF32_Hi(vxc_ushort8 src)\n\
{\n\
    vxc_half8 srcHalf;\n\
    float4 dst;\n\
\n\
    _viv_asm(COPY, srcHalf, src, 16);\n\
    VXC_DP4x4(dst, srcHalf, srcHalf, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvFP16toFP32_Hi_4x4);\n\
    return dst;\n\
}\n\
\n\
float4 BF16toF32_Lo(vxc_ushort8 src)\n\
{\n\
    vxc_ushort8 srcA;\n\
    float4 dst;\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(srcA, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, dst, srcA, 16);\n\
\n\
    return dst;\n\
}\n\
\n\
float4 BF16toF32_Hi(vxc_ushort8 src)\n\
{\n\
    vxc_ushort8 srcA;\n\
    float4 dst;\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    VXC_DP2x8(srcA, src, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, dst, srcA, 16);\n\
\n\
    return dst;\n\
}\n\
\n\
vxc_ushort8 F32toF16(float4 src0, float4 src1)\n\
{\n\
    half4 srcHalf0, srcHalf1;\n\
    vxc_half8 dst0;\n\
    vxc_ushort8 dst;\n\
\n\
    _viv_asm(CONV, srcHalf0, src0);\n\
    _viv_asm(CONV, srcHalf1, src1);\n\
\n\
    VXC_DP2x8(dst0, srcHalf0, srcHalf1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractHalf8_2x8);\n\
    _viv_asm(COPY, dst, dst0, 16);\n\
    return dst;\n\
}\n\
\n\
vxc_ushort8 F32toBF16(float4 src0, float4 src1)\n\
{\n\
    vxc_ushort8 srcA, srcB;\n\
    vxc_ushort8 dst;\n\
\n\
    _viv_asm(COPY, srcA, src0, 16);\n\
    _viv_asm(COPY, srcB, src1, 16);\n\
    VXC_DP2x8(dst, srcA, srcB, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniPackedBF16_2x8);\n\
    return dst;\n\
}\n\
\n\
vxc_char8 F32toI8(float4 src0, float4 src1)\n\
{\n\
    int4 srcInt0, srcInt1;\n\
    vxc_char8 dst;\n\
\n\
    src0 *= output_scale;\n\
    src1 *= output_scale;\n\
    _viv_asm(CONV_RTE, srcInt0, src0);\n\
    _viv_asm(CONV_RTE, srcInt1, src1);\n\
\n\
    VXC_DP2x8(dst, srcInt0, srcInt1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInteger_2x8);\n\
    return dst;\n\
}\n\
\n\
vxc_short8 F32toI16(float4 src0, float4 src1)\n\
{\n\
    int4 srcInt0, srcInt1;\n\
    vxc_short8 dst;\n\
\n\
    src0 *= output_scale;\n\
    src1 *= output_scale;\n\
    _viv_asm(CONV_RTE, srcInt0, src0);\n\
    _viv_asm(CONV_RTE, srcInt1, src1);\n\
\n\
    VXC_DP2x8(dst, srcInt0, srcInt1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInteger_2x8);\n\
    return dst;\n\
}\n\
\n\
vxc_uchar8 F32toU8(float4 src0, float4 src1)\n\
{\n\
    int4 srcInt0, srcInt1;\n\
    vxc_uchar8 dst;\n\
\n\
    src0 = src0 * output_scale + outputZP;\n\
    src1 = src1 * output_scale + outputZP;\n\
    _viv_asm(CONV_RTE, srcInt0, src0);\n\
    _viv_asm(CONV_RTE, srcInt1, src1);\n\
\n\
    VXC_DP2x8(dst, srcInt0, srcInt1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniExtractInteger_2x8);\n\
    return dst;\n\
}\n\
\n\
\n\
#define TENSOR_KERAS_RELU(src_type_name, dst_type_name, tensor_dims, image_type, \\\n\
    convert2FP32_Func, convert2DstType_Func, src_type, dst_type) \\\n\
__kernel void relu_keras_##src_type_name##to##dst_type_name##tensor_dims( \\\n\
__read_only  image2d_array_t  input, \\\n\
__write_only image2d_array_t  output, \\\n\
                    float     alpha, \\\n\
                    float     max_value, \\\n\
                    float     threshold \\\n\
                    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    VXC_Read##image_type(src, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 dataA = convert2FP32_Func##_Lo(src); \\\n\
    float4 dataB = convert2FP32_Func##_Hi(src); \\\n\
    float4 dataA0 = dataA < threshold ? threshold : dataA; \\\n\
    dataA0 = dataA0 > max_value ? max_value : dataA0; \\\n\
    float4 dataB0 = dataB < threshold ? threshold : dataB; \\\n\
    dataB0 = dataB0 > max_value ? max_value : dataB0; \\\n\
    float4 dataA1 = dataA * alpha - offset; \\\n\
    float4 dataB1 = dataB * alpha - offset; \\\n\
    float4 dst0 = dataA  < threshold ? dataA1 : dataA0; \\\n\
    float4 dst1 = dataB  < threshold ? dataB1 : dataB0; \\\n\
    dst_type result = convert2DstType_Func(dst0, dst1); \\\n\
    VXC_Write##image_type(output, coord, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
TENSOR_KERAS_RELU(F16,  F16,  _3D, Image2DArray, F16toF32,  F32toF16,  vxc_ushort8, vxc_ushort8)\n\
TENSOR_KERAS_RELU(F16,  I16,  _3D, Image2DArray, F16toF32,  F32toI16,  vxc_ushort8, vxc_short8)\n\
TENSOR_KERAS_RELU(F16,  I8,   _3D, Image2DArray, F16toF32,  F32toI8,   vxc_ushort8, vxc_char8)\n\
TENSOR_KERAS_RELU(F16,  U8,   _3D, Image2DArray, F16toF32,  F32toU8,   vxc_ushort8, vxc_uchar8)\n\
TENSOR_KERAS_RELU(BF16, BF16, _3D, Image2DArray, BF16toF32, F32toBF16, vxc_ushort8, vxc_ushort8)\n\
\n\
TENSOR_KERAS_RELU(I16,  I16,  _3D, Image2DArray, I16toF32,  F32toI16,  vxc_short8,  vxc_short8)\n\
TENSOR_KERAS_RELU(I16,  F16,  _3D, Image2DArray, I16toF32,  F32toF16,  vxc_short8,  vxc_ushort8)\n\
TENSOR_KERAS_RELU(I8,   I8,   _3D, Image2DArray, I8toF32,   F32toI8,   vxc_char8,   vxc_char8)\n\
TENSOR_KERAS_RELU(I8,   F16,  _3D, Image2DArray, I8toF32,   F32toF16,  vxc_char8,   vxc_ushort8)\n\
TENSOR_KERAS_RELU(U8,   U8,   _3D, Image2DArray, U8toF32,   F32toU8,   vxc_uchar8,  vxc_uchar8)\n\
TENSOR_KERAS_RELU(U8,   F16,  _3D, Image2DArray, U8toF32,   F32toF16,  vxc_uchar8,  vxc_ushort8)\n\
\n\
TENSOR_KERAS_RELU(F16,  F16,  _2D, Image,        F16toF32,  F32toF16,  vxc_ushort8, vxc_ushort8)\n\
TENSOR_KERAS_RELU(F16,  I16,  _2D, Image,        F16toF32,  F32toI16,  vxc_ushort8, vxc_short8)\n\
TENSOR_KERAS_RELU(F16,  I8,   _2D, Image,        F16toF32,  F32toI8,   vxc_ushort8, vxc_char8)\n\
TENSOR_KERAS_RELU(F16,  U8,   _2D, Image,        F16toF32,  F32toU8,   vxc_ushort8, vxc_uchar8)\n\
TENSOR_KERAS_RELU(BF16, BF16, _2D, Image,        BF16toF32, F32toBF16, vxc_ushort8, vxc_ushort8)\n\
TENSOR_KERAS_RELU(I16,  I16,  _2D, Image,        I16toF32,  F32toI16,  vxc_short8,  vxc_short8)\n\
TENSOR_KERAS_RELU(I16,  F16,  _2D, Image,        I16toF32,  F32toF16,  vxc_short8,  vxc_ushort8)\n\
TENSOR_KERAS_RELU(I8,   I8,   _2D, Image,        I8toF32,   F32toI8,   vxc_char8,   vxc_char8)\n\
TENSOR_KERAS_RELU(I8,   F16,  _2D, Image,        I8toF32,   F32toF16,  vxc_char8,   vxc_ushort8)\n\
TENSOR_KERAS_RELU(U8,   U8,   _2D, Image,        U8toF32,   F32toU8,   vxc_uchar8,  vxc_uchar8)\n\
TENSOR_KERAS_RELU(U8,   F16,  _2D, Image,        U8toF32,   F32toF16,  vxc_uchar8,  vxc_ushort8)\n\
"; /* end of relu_keras_vx*/

static const char repeat_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniIntegralHorAcc_4x4;\n\
_viv_uniform VXC_512Bits uniExtract1to8Short_2x8;\n\
_viv_uniform int width;\n\
\n\
// workgroup size is 32\n\
__kernel void preprocess_start_idx(image2d_t input, image2d_t output)\n\
{\n\
    int lidx = get_local_id(0);\n\
    __local int lcl_sum[32];\n\
    __local int last_round[1];\n\
    Image img = create_image_from_image2d(input, 4);\n\
    Image dst = create_image_from_image2d(output, 4);\n\
    __global int* index_ptr = (__global int*)img.ptr + get_global_id(0);\n\
    __global int* output_org = (__global int*)dst.ptr;\n\
    __global int* output_ptr = output_org + get_global_id(0) + 1;\n\
\n\
    if (lidx == 0)\n\
    {\n\
        last_round[0] = 0;\n\
        output_org[0] = 0;\n\
    }\n\
    int4 accSum0, accSum1, accSum2, accSum3;\n\
\n\
    for(int i = 0; i < width; i += 512)\n\
    {\n\
        int4 data0 = vload4(0, index_ptr + i);\n\
        int4 data1 = vload4(1, index_ptr + i);\n\
        int4 data2 = vload4(2, index_ptr + i);\n\
        int4 data3 = vload4(3, index_ptr + i);\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
        int prevSum = last_round[0];\n\
\n\
        VXC_DP4x4(accSum0, data0, data0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        VXC_DP4x4(accSum1, data1, data1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        VXC_DP4x4(accSum2, data2, data2, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        VXC_DP4x4(accSum3, data3, data3, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniIntegralHorAcc_4x4);\n\
        accSum1 += accSum0.w;\n\
        accSum2 += accSum1.w;\n\
        accSum3 += accSum2.w;\n\
\n\
        lcl_sum[lidx] = accSum3.w;\n\
        barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
        for(int j = 0; j < lidx; j++)\n\
        {\n\
            prevSum += lcl_sum[j];\n\
        }\n\
        accSum0 += prevSum;\n\
        accSum1 += prevSum;\n\
        accSum2 += prevSum;\n\
        accSum3 += prevSum;\n\
        if(lidx == 31)\n\
        {\n\
            last_round[0] = accSum3.w;\n\
        }\n\
        vstore4(accSum0, 0, output_ptr + i);\n\
        vstore4(accSum1, 1, output_ptr + i);\n\
        vstore4(accSum2, 2, output_ptr + i);\n\
        vstore4(accSum3, 3, output_ptr + i);\n\
    }\n\
}\n\
\n\
__kernel void repeat_I16_axis0(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2));\n\
    vxc_short8 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(1)];\n\
    int start = index_ptr[get_global_id(1)];\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int end = len + start;\n\
    coord.w = get_global_id(2);\n\
\n\
    for(coord.y = start; coord.y < end; coord.y++)\n\
    {\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, \\\n\
                VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void repeat_I16_axis2(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(2)];\n\
    int start = index_ptr[get_global_id(2)];\n\
    int end = len + start;\n\
\n\
    for(coord.z = start; coord.z < end; coord.z++)\n\
    {\n\
        VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define REPEAT_1D(src0_type_name, data_type) \\\n\
__kernel void repeat_##src0_type_name##_1D( \\\n\
    image2d_t  input0, image2d_t  input1, image2d_t  input2, \\\n\
    image2d_t  output, int axis) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), 0); \\\n\
    data_type src0; \\\n\
    VXC_ReadImage(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
             VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    Image img2 = create_image_from_image2d(input2, 4); \\\n\
    __global int* len_ptr = (__global int*)img1.ptr; \\\n\
    __global int* index_ptr = (__global int*)img2.ptr; \\\n\
    int len = len_ptr[get_global_id(0)]; \\\n\
    int start = index_ptr[get_global_id(0)]; \\\n\
 \\\n\
    int iter = len >> 3; \\\n\
    int res = len & 7; \\\n\
    int end = start + iter * 8; \\\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8); \\\n\
    for(coord.x = start; coord.x < end; coord.x+=8) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
 \\\n\
    if(res == 7) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 6, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 6) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 5, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 5) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 4, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 4) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 3) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 2, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 2) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else if(res == 1) \\\n\
    { \\\n\
        VXC_WriteImage(output, coord, src0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
REPEAT_1D(U8,  vxc_uchar16)\n\
REPEAT_1D(I16, vxc_short8)\n\
\n\
__kernel void repeat_U8_axis0(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_uchar16 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(1)];\n\
    int start = index_ptr[get_global_id(1)];\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int end = len + start;\n\
    coord.w = get_global_id(2);\n\
\n\
    for(coord.y = start; coord.y < end; coord.y++)\n\
    {\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, \\\n\
                VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
__kernel void repeat_U8_axis2(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_uchar16 src0;\n\
    VXC_ReadImage2DArray(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(2)];\n\
    int start = index_ptr[get_global_id(2)];\n\
    int end = len + start;\n\
\n\
    for(coord.z = start; coord.z < end; coord.z++)\n\
    {\n\
        VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of repeat_vx*/

static const char repeat_axis1_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtract1to8Short_2x8;\n\
\n\
#define REPEAT_RES(end_pos) \\\n\
coord.y = gidy; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src0, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src1, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src3, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src4, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src5, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src6, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0)); \\\n\
coord.y++; \\\n\
VXC_OP4_NoDest(img_store_3d, output, coord, src7, VXC_MODIFIER(0, end_pos, 0,VXC_RM_TowardZero, 0));\n\
\n\
__kernel void repeat_I16_axis1(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), gidy, get_global_id(2), get_global_id(2));\n\
    vxc_short8 src0, src1, src2, src3, src4, src5, src6, src7;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
\n\
    VXC_OP4(img_load_3d, src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input0, coord, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input0, coord, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input0, coord, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src4, input0, coord, VXC_5BITOFFSET_XY(0, 4),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src5, input0, coord, VXC_5BITOFFSET_XY(0, 5),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src6, input0, coord, VXC_5BITOFFSET_XY(0, 6),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src7, input0, coord, VXC_5BITOFFSET_XY(0, 7),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(0)];\n\
    int start = index_ptr[get_global_id(0)];\n\
\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int iter = len >> 3;\n\
    int res = len & 7;\n\
    coord.x = start;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src2, src2, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src3, src3, src3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src4, src4, src4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src5, src5, src5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src6, src6, src6, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src7, src7, src7, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        coord.y = gidy;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src1, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src3, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src4, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src5, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src6, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src7, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
    }\n\
\n\
    if(res == 7)\n\
    {\n\
        REPEAT_RES(6)\n\
    }\n\
    else if(res == 6)\n\
    {\n\
        REPEAT_RES(5)\n\
    }\n\
    else if(res == 5)\n\
    {\n\
        REPEAT_RES(4)\n\
    }\n\
    else if(res == 4)\n\
    {\n\
        REPEAT_RES(3)\n\
    }\n\
    else if(res == 3)\n\
    {\n\
        REPEAT_RES(2)\n\
    }\n\
    else if(res == 2)\n\
    {\n\
        REPEAT_RES(1)\n\
    }\n\
    else if(res == 1)\n\
    {\n\
        REPEAT_RES(0)\n\
    }\n\
}\n\
\n\
__kernel void repeat_U8_axis1(\n\
    image2d_array_t  input0, image2d_t  input1, image2d_t  input2,\n\
    image2d_array_t  output, int axis)\n\
{\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), gidy, get_global_id(2), get_global_id(2));\n\
    vxc_uchar16 src0, src1, src2, src3, src4, src5, src6, src7;\n\
\n\
    int8 input_desc, output_desc;\n\
    _viv_asm(COPY, input_desc, input0, sizeof(input_desc));\n\
    int baseAddr_a = (int)get_global_id(2) * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord.z, baseAddr_a);\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    int baseAddr = (int)get_global_id(2) * output_desc.s4 + output_desc.s0;\n\
\n\
    VXC_OP4(img_load_3d, src0, input0, coord, VXC_5BITOFFSET_XY(0, 0),\n\
             VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input0, coord, VXC_5BITOFFSET_XY(0, 1),\n\
             VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input0, coord, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input0, coord, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src4, input0, coord, VXC_5BITOFFSET_XY(0, 4),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src5, input0, coord, VXC_5BITOFFSET_XY(0, 5),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src6, input0, coord, VXC_5BITOFFSET_XY(0, 6),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src7, input0, coord, VXC_5BITOFFSET_XY(0, 7),\n\
            VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 4);\n\
    __global int* len_ptr = (__global int*)img1.ptr;\n\
    __global int* index_ptr = (__global int*)img2.ptr;\n\
\n\
    int len = len_ptr[get_global_id(0)];\n\
    int start = index_ptr[get_global_id(0)];\n\
\n\
    _viv_asm(MOV, coord.z, baseAddr);\n\
    int iter = len >> 3;\n\
    int res = len & 7;\n\
    coord.x = start;\n\
\n\
    VXC_DP2x8(src0, src0, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src1, src1, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src2, src2, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src3, src3, src3, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src4, src4, src4, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src5, src5, src5, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src6, src6, src6, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
    VXC_DP2x8(src7, src7, src7, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtract1to8Short_2x8);\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        coord.y = gidy;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src0, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src1, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src2, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src3, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src4, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src5, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src6, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord, src7, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0));\n\
        coord.x += 8;\n\
    }\n\
\n\
    if(res == 7)\n\
    {\n\
        REPEAT_RES(6)\n\
    }\n\
    else if(res == 6)\n\
    {\n\
        REPEAT_RES(5)\n\
    }\n\
    else if(res == 5)\n\
    {\n\
        REPEAT_RES(4)\n\
    }\n\
    else if(res == 4)\n\
    {\n\
        REPEAT_RES(3)\n\
    }\n\
    else if(res == 3)\n\
    {\n\
        REPEAT_RES(2)\n\
    }\n\
    else if(res == 2)\n\
    {\n\
        REPEAT_RES(1)\n\
    }\n\
    else if(res == 1)\n\
    {\n\
        REPEAT_RES(0)\n\
    }\n\
}\n\
\n\
"; /* end of repeat_axis1_vx*/

static const char resize_1d_bilinear_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_odd_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_even_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_1d_bilinear_BF16toBF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x      = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x         = (convert_float4(coord_x) + half_pixel_value)  * scale_x - half_pixel_value;\n\
    float4 left_x_f     = floor(in_x);\n\
    float4 x_lerp       = in_x - left_x_f;\n\
    int4   left_x_idx   = convert_int4(left_x_f);\n\
\n\
    vxc_short8 top;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_ushort8 src;\n\
        float4 left4;\n\
        float4 right4;\n\
        float4 dst4;\n\
\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8);\n\
        _viv_asm(COPY, right4, src, 16);\n\
        VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8);\n\
        _viv_asm(COPY, left4,  src, 16);\n\
        right4    -= left4;\n\
        dst4       = right4 * x_lerp + left4;\n\
        vxc_ushort8 tmp, dst;\n\
        _viv_asm(COPY, tmp, dst4, 16);\n\
        dst.s0123 = tmp.s1357;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_BF16toBF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.y ++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_ushort8 dst_tmp;\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, left4, dst_tmp, 16);\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, right4, dst_tmp, 16);\n\
        right4     -= left4;\n\
        float4 dst4 = right4 * x_lerp + left4;\n\
\n\
        vxc_ushort8 tmp, dst;\n\
        _viv_asm(COPY, tmp, dst4, 16);\n\
        dst.s0123 = tmp.s1357;\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
"; /* end of resize_1d_bilinear_BF16_vx*/

static const char resize_1d_bilinear_DOWN_NX_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResizeNxDown_2x8;\n\
_viv_uniform int out_height;\n\
\n\
#define RESIZE_1D_NX_DOWN_8BIT_SAME_PROCESS(read_type, data_type) \\\n\
    read_type read_data, save_data; \\\n\
    data_type in0, result; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    while (coord_out.y < out_height) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, read_data, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in0, read_data, 16); \\\n\
        VXC_DP2x8(result, in0, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResizeNxDown_2x8); \\\n\
        _viv_asm(COPY, save_data, result, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, save_data, \\\n\
            VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y++; \\\n\
        coord_out.y++; \\\n\
    } \\\n\
\n\
#define RESIZE_1D_2X_DOWN_8BIT_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_8BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_8BIT_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_DOWN_8BIT_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
\n\
\n\
\n\
#define RESIZE_1D_2X_DOWN_8BIT_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_8BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_8BIT_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_DOWN_8BIT_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
\n\
\n\
#define RESIZE_1D_NX_DOWN_16BIT_SAME_PROCESS(read_type, data_type) \\\n\
    read_type read_data, read_data1, save_data; \\\n\
    data_type in0, in1, result; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    while (coord_out.y < out_height) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, read_data, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in0, read_data, 16); \\\n\
        VXC_OP4(img_load_3d, read_data1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in1, read_data1, 16); \\\n\
        VXC_DP2x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResizeNxDown_2x8); \\\n\
        _viv_asm(COPY, save_data, result, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, save_data, \\\n\
            VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y++; \\\n\
        coord_out.y++; \\\n\
    } \\\n\
\n\
#define RESIZE_1D_2X_DOWN_16BIT_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_16BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_16BIT_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_DOWN_16BIT_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
\n\
#define RESIZE_1D_2X_DOWN_16BIT_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_DOWN_2X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x << 1; \\\n\
    RESIZE_1D_NX_DOWN_16BIT_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_DOWN_16BIT_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_DOWN_16BIT_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
"; /* end of resize_1d_bilinear_DOWN_NX_vx*/

static const char resize_1d_bilinear_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform VXC_512Bits uniConvertFp2FP32_left_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFp2FP32_right_4x4;\n\
\n\
__kernel void resize_1d_bilinear_F16toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 left4;\n\
    float4 right4;\n\
\n\
    vxc_ushort8 src, result;\n\
    vxc_half8 src_half, dst;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        _viv_asm(COPY, src_half, src, 16);\n\
\n\
        VXC_DP4x4(left4,  src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFp2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFp2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        half4 tmp;\n\
        _viv_asm(CONV, tmp, dst4);\n\
        VXC_DP2x8(dst, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, result, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
\n\
}\n\
\n\
__kernel void resize_1d_bilinear_F16toU8_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 left4;\n\
    float4 right4;\n\
\n\
    vxc_ushort8 src;\n\
    vxc_uchar8 result;\n\
    vxc_half8 src_half, dst;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        _viv_asm(COPY, src_half, src, 16);\n\
\n\
        VXC_DP4x4(left4,  src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFp2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFp2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_F16toF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_half8 top;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
\n\
        coord_in.y ++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_part1_4x4);\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        half4 tmp;\n\
        _viv_asm(CONV, tmp, dst4);\n\
        VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, dst0, top, 16);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
"; /* end of resize_1d_bilinear_F16_vx*/

static const char resize_1d_bilinear_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_4x4;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_left_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_right_4x4;\n\
\n\
__kernel void resize_1d_bilinear_I16toI16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
\n\
    vxc_short8 top;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        coord_in.y ++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_part1_4x4);\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4         = dst4 * dfpScale;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_I16toI16_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
    vxc_short8 result;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4         = dst4 * dfpScale;\n\
\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
"; /* end of resize_1d_bilinear_I16_vx*/

static const char resize_1d_bilinear_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_4x4;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_left_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDFP2FP32_right_4x4;\n\
\n\
__kernel void resize_1d_bilinear_I8toI8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    vxc_uchar16 src0, dst0;\n\
\n\
    vxc_char16 top;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(dst0, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
\n\
        coord_in.y++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        VXC_DP4x4(left4, top, top, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_4x4);\n\
        VXC_DP4x4(right4, top, top, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_part1_4x4);\n\
\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
\n\
        dst4         = dst4 * dfpScale;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_I8toI8_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_char16 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
    vxc_char16 result;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_left_4x4);\n\
        VXC_DP4x4(right4, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDFP2FP32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4         = dst4 * dfpScale;\n\
\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
\n\
}\n\
"; /* end of resize_1d_bilinear_I8_vx*/

static const char resize_1d_bilinear_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_right_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_1d_bilinear_U8toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4 *=  uint8Scale;\n\
        half4 dst;\n\
        _viv_asm(CONV, dst, dst4);\n\
        vxc_short8 dst_short;\n\
        _viv_asm(COPY, dst_short, dst, 16);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst_short,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_U8toU8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
\n\
    vxc_uchar16 src0, src1;\n\
\n\
    vxc_uchar16 top;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(top, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        float4 left4;\n\
        float4 right4;\n\
\n\
        unsigned char inputZP;\n\
        _viv_asm(COPY, inputZP, input_ZP, 4);\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_part1_4x4);\n\
\n\
        float4 dst4 = right4 * x_lerp + left4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
\n\
__kernel void resize_1d_bilinear_U8toU8_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
    float4 left4;\n\
    float4 right4;\n\
    vxc_uchar16 result;\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
\n\
    do\n\
    {\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.y;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.z;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
        coord_in.x = left_x_idx.w;\n\
        VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, src, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_right_4x4);\n\
        right4      -= left4;\n\
        float4 dst4  = right4 * x_lerp + left4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y++;\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
\n\
}\n\
"; /* end of resize_1d_bilinear_U8_vx*/

static const char resize_1d_bilinear_U8_opt_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float scale_x;\n\
_viv_uniform int out_height;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4x4;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_1d_bilinear_U8toU8_UP_opt\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers,\n\
    __read_only  image2d_array_t   scale\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_x - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    int4   right_x_idx = left_x_idx + 1;\n\
\n\
    vxc_uchar16 src0;\n\
\n\
    vxc_uchar16 src_mask;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    vxc_ushort8 lerp_0;\n\
    vxc_half8 lerp;\n\
\n\
    int2 coord = (int2)(coord_out.x * 2, 0);\n\
    VXC_ReadImage(lerp_0, scale, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, lerp, lerp_0, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    do\n\
    {\n\
        VXC_BitExtract(src_mask, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.y++;\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_uchar16 dst;\n\
        VXC_DP4x4(dst, src_mask, lerp,\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniBilinear_4x4);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
        coord_out.y ++;\n\
    } while (coord_out.y < out_height);\n\
}\n\
"; /* end of resize_1d_bilinear_U8_opt_vx*/

static const char resize_1d_bilinear_UP_NX_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResizeNxUp_2x8;\n\
_viv_uniform int out_height;\n\
\n\
#define RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
    read_type read_data, save_data; \\\n\
    data_type in0, result; \\\n\
 \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in.w, baseAddr); \\\n\
 \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord_out.w, baseAddr); \\\n\
 \\\n\
    while (coord_out.y < out_height) \\\n\
    { \\\n\
        VXC_OP4(img_load_3d, read_data, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, in0, read_data, 16); \\\n\
        VXC_DP2x8(result, in0, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResizeNxUp_2x8); \\\n\
        _viv_asm(COPY, save_data, result, 16); \\\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, save_data, \\\n\
            VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
        coord_in.y++; \\\n\
        coord_out.y++; \\\n\
    } \\\n\
\n\
#define RESIZE_1D_2X_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_2X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x = (coord_out.x * 2 - 1) >> 2; \\\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_2X_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_2X_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_2X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x >> 1; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_2X_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_2X_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_2X_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_2X_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_4X_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_4X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x = (coord_out.x * 2 - 3) >> 3; \\\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_4X_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_4X_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_4X_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_4X_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_4X_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_4X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x >> 2; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_4X_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_4X_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_4X_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_4X_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_8X_HALF_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_8X_HALF_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x = (coord_out.x * 2 - 7) >> 4; \\\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_8X_HALF_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_8X_HALF_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_8X_HALF_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_8X_HALF_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
#define RESIZE_1D_8X_SAME(name0, name1, read_type, data_type) \\\n\
__kernel void resize_1d_bilinear_##name0##to##name1##_UP_8X_SAME \\\n\
    ( \\\n\
    __read_only  image2d_array_t   input, \\\n\
    __write_only image2d_array_t   output, \\\n\
                             int   scale_type \\\n\
    ) \\\n\
{ \\\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    int4 coord_in   = (int4)(get_global_id(0), 0, get_global_id(1), 0); \\\n\
    coord_in.x =  coord_out.x >> 3; \\\n\
    RESIZE_1D_NX_SAME_PROCESS(read_type, data_type) \\\n\
}\n\
\n\
RESIZE_1D_8X_SAME(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
RESIZE_1D_8X_SAME(I8,  I8,  vxc_char16,  vxc_char16)\n\
RESIZE_1D_8X_SAME(I16, I16, vxc_short8,  vxc_short8)\n\
RESIZE_1D_8X_SAME(F16, F16, vxc_short8,  vxc_half8)\n\
\n\
\n\
"; /* end of resize_1d_bilinear_UP_NX_vx*/

static const char resize_1d_nearest_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniMultiplyAndPostShift_2x8;\n\
_viv_uniform float scale_x;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform float round_value;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4   coord_x    = coord_out.xxxx + (int4)(0, 1, 2, 3); \\\n\
    float4 in_x       = (convert_float4(coord_x) + half_pixel_value) * scale_x + round_value; \\\n\
    int4   in_x_idx   = convert_int4(in_x); \\\n\
\n\
\n\
__kernel void resize_1d_nearest_F16toF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniGetExtractData_2x8;\n\
__kernel void resize_1d_nearest_F16toF16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_2x8;\n\
__kernel void resize_1d_nearest_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_char16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_I8toI8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst0;\n\
    vxc_char16 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_U8toU8\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(src, src, multiplier, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_U8toU8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(dst, dst, multiplier, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_I16toI16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_1d_nearest_I16toI16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_short8 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, coord_out.y, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_1d_nearest_vx*/

static const char resize_bilinear_BF16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_odd_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_even_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_BF16toBF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x      = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x         = (convert_float4(coord_x) + half_pixel_value)  * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f     = floor(in_x);\n\
    float4 x_lerp       = in_x - left_x_f;\n\
    int4   left_x_idx   = convert_int4(left_x_f);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float  y_lerp       = in_y - top_y_f;\n\
    int    top_y_idx    = convert_int(top_y_f);\n\
    vxc_short8 top;\n\
    vxc_short8 bottom;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 src;\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
    float4 dst4;\n\
\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8);\n\
    _viv_asm(COPY, right4, src, 16);\n\
    VXC_DP2x8(src, top, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8);\n\
    _viv_asm(COPY, left4,  src, 16);\n\
    right4    -= left4;\n\
    top4       = right4 * x_lerp + left4;\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_even_2x8);\n\
    _viv_asm(COPY, right4, src, 16);\n\
    VXC_DP2x8(src, bottom, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_odd_2x8);\n\
    _viv_asm(COPY, left4,  src, 16);\n\
    right4    -= left4;\n\
    bottom4    = right4 * x_lerp + left4;\n\
    bottom4   -= top4;\n\
    dst4       = bottom4 * y_lerp + top4;\n\
    vxc_ushort8 tmp, dst;\n\
    _viv_asm(COPY, tmp, dst4, 16);\n\
    dst.s0123 = tmp.s1357;\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_BF16toBF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_ushort8 src0, src1, src2, src3, dst0, dst1;\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_ushort8 dst_tmp;\n\
\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, left4, dst_tmp, 16);\n\
        VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, right4, dst_tmp, 16);\n\
        right4     -= left4;\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
        _viv_asm(COPY, left4, dst_tmp, 16);\n\
        VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
        _viv_asm(COPY, right4, dst_tmp, 16);\n\
        right4     -= left4;\n\
        bottom4     = right4 * x_lerp + left4;\n\
\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
        vxc_ushort8 tmp, dst;\n\
        _viv_asm(COPY, tmp, dst4, 16);\n\
        dst.s0123 = tmp.s1357;\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 dst_tmp;\n\
    VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, left4, dst_tmp, 16);\n\
    VXC_DP2x8(dst_tmp, dst0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, right4, dst_tmp, 16);\n\
    right4     -= left4;\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8);\n\
    _viv_asm(COPY, left4, dst_tmp, 16);\n\
    VXC_DP2x8(dst_tmp, dst1, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8);\n\
    _viv_asm(COPY, right4, dst_tmp, 16);\n\
    right4     -= left4;\n\
    bottom4     = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    vxc_ushort8 tmp, dst;\n\
    _viv_asm(COPY, tmp, dst4, 16);\n\
    dst.s0123 = tmp.s1357;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_BF16_vx*/

static const char resize_bilinear_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtactHalf8_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniFp16toFp32_part1_4x4;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_F16toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_short8 top_short, bottom_short, dst;\n\
    vxc_half8  top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top,    top_short, 16);\n\
    _viv_asm(COPY, bottom, bottom_short, 16);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4       = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4    = right4 * x_lerp + left4;\n\
    bottom4   -= top4;\n\
    float4 dst4       = bottom4 * y_lerp + top4;\n\
\n\
    half4 tmp;\n\
    _viv_asm(CONV, tmp, dst4);\n\
    VXC_DP2x8(result, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
    _viv_asm(COPY, dst, result, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_F16toU8_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_short8 top_short, bottom_short;\n\
    vxc_half8  top, bottom;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom_short, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top,    top_short, 16);\n\
    _viv_asm(COPY, bottom, bottom_short, 16);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    vxc_uchar8 dst_uchar;\n\
    VXC_DP2x8(dst_uchar, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst_uchar,\n\
        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_F16toF16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
\n\
    vxc_ushort8 src0, src1, src2, src3, dst0, dst1;\n\
    vxc_half8 top;\n\
    vxc_half8 bottom;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
        _viv_asm(COPY, bottom, dst1, 16);\n\
\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
        top4        = right4 * x_lerp + left4;\n\
        VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        half4 tmp;\n\
        _viv_asm(CONV, tmp, dst4);\n\
        VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
        _viv_asm(COPY, dst0, top, 16);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top, dst0, 16);\n\
    _viv_asm(COPY, bottom, dst1, 16);\n\
\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniFp16toFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    half4 tmp;\n\
    _viv_asm(CONV, tmp, dst4);\n\
    VXC_DP2x8(top, tmp, tmp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtactHalf8_2x8);\n\
    _viv_asm(COPY, dst0, top, 16);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_F16_vx*/

static const char resize_bilinear_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_I16toI16_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_ushort8 src0, src1, src2, src3, dst0, dst1;\n\
\n\
    vxc_short8 top;\n\
    vxc_short8 bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 16;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
        _viv_asm(COPY, bottom, dst1, 16);\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src3, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 1),\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP4x4(left4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        dst4         = dst4 * dfpScale;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src2, src3, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top, dst0, 16);\n\
    _viv_asm(COPY, bottom, dst1, 16);\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * dfpScale;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
}\n\
\n\
__kernel void resize_bilinear_I16toI16_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_short8 top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    VXC_DP4x4(left4, top, top, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * dfpScale;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
        VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of resize_bilinear_I16_vx*/

static const char resize_bilinear_I8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniDFPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniRightSubLeft_4x4;\n\
_viv_uniform float dfpScale;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_I8toI8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_uchar16 src0, src1, dst0, dst1;\n\
\n\
    vxc_char16 top;\n\
    vxc_char16 bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(dst0, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(dst1, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, top, dst0, 16);\n\
        _viv_asm(COPY, bottom, dst1, 16);\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        VXC_DP4x4(left4, top, top, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP4x4(left4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        dst4         = dst4 * dfpScale;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(dst0, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(dst1, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, top, dst0, 16);\n\
    _viv_asm(COPY, bottom, dst1, 16);\n\
    VXC_DP4x4(left4, top, top, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * dfpScale;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_I8toI8_DOWN\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_char16 top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    VXC_DP4x4(left4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDFPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniRightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4         = dst4 * dfpScale;\n\
\n\
    int4 dst     = convert_int4_rte(dst4);\n\
\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_I8_vx*/

static const char resize_bilinear_U8_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZPtoFp32_left_4x4;\n\
_viv_uniform VXC_512Bits uniU8RightSubLeft_4x4;\n\
_viv_uniform VXC_512Bits uniExtact8Bit_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform int input_ZP;\n\
_viv_uniform float uint8Scale;\n\
_viv_uniform float output_ZP;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_U8toF16_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_uchar16 top, bottom;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4 *=  uint8Scale;\n\
\n\
    half4 dst;\n\
    _viv_asm(CONV, dst, dst4);\n\
\n\
    vxc_short8 dst_short;\n\
    _viv_asm(COPY, dst_short, dst, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst_short.s0246,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_U8toU8_UP\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float4 right_x_f   = ceil(in_x);\n\
    int4   right_x_idx = convert_int4(right_x_f);\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_uchar16 src0, src1;\n\
\n\
    vxc_uchar16 top;\n\
    vxc_uchar16 bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData, \\\n\
    VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(top, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(bottom, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
                VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        unsigned char inputZP;\n\
        _viv_asm(COPY, inputZP, input_ZP, 4);\n\
        VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        top4        = right4 * x_lerp + left4;\n\
\n\
        VXC_DP4x4(left4, bottom, inputZP, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
        VXC_DP4x4(right4, bottom, bottom, \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
        bottom4      = right4 * x_lerp + left4;\n\
        bottom4     -= top4;\n\
        float4 dst4  = bottom4 * y_lerp + top4;\n\
        dst4         = dst4 * uint8Scale + output_ZP;\n\
        int4 dst     = convert_int4_rte(dst4);\n\
        VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(top, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(bottom, src1, src1, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, top, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, bottom, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
    VXC_DP2x8(top, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, top, VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_bilinear_U8toU8_DOWN\n\
    (\n\
    __read_only     image2d_array_t input,\n\
    __write_only    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
    float4 left_x_f    = floor(in_x);\n\
    float4 x_lerp      = in_x - left_x_f;\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
    float  top_y_f     = floor(in_y);\n\
    float  y_lerp      = in_y - top_y_f;\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
    vxc_uchar16 top, bottom, result;\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.y;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.z;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(4, 5, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = left_x_idx.w;\n\
    VXC_OP4(img_load_3d, top, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, bottom, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(6, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    float4 left4;\n\
    float4 right4;\n\
    float4 top4;\n\
    float4 bottom4;\n\
\n\
    unsigned char inputZP;\n\
    _viv_asm(COPY, inputZP, input_ZP, 4);\n\
    VXC_DP4x4(left4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, top, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    top4        = right4 * x_lerp + left4;\n\
\n\
    VXC_DP4x4(left4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8SubZPtoFp32_left_4x4);\n\
    VXC_DP4x4(right4, bottom, inputZP, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniU8RightSubLeft_4x4);\n\
    bottom4      = right4 * x_lerp + left4;\n\
\n\
    bottom4     -= top4;\n\
    float4 dst4  = bottom4 * y_lerp + top4;\n\
\n\
    dst4         = dst4 * uint8Scale + output_ZP;\n\
    int4 dst     = convert_int4_rte(dst4);\n\
\n\
    VXC_DP2x8(result, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniExtact8Bit_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_U8_vx*/

static const char resize_bilinear_U8_half_pixel_centers_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResize2xUp_0_4x8;\n\
_viv_uniform VXC_512Bits uniResize2xUp_1_4x8;\n\
_viv_uniform int out_height;\n\
\n\
__kernel void resize_bilinear_U8toU8_SAME_2x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 1) >> 2;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, tmp, result;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_0_4x8);\n\
        VXC_DP4x8(result, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize2xUp_1_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, result,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        coord_out.y++;\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize4xUp_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l01_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l10_4x8;\n\
_viv_uniform VXC_512Bits uniResize4xUp_l11_4x8;\n\
__kernel void resize_bilinear_U8toU8_SAME_4x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, get_global_id(1), 0);\n\
    coord_in.x = (coord_out.x * 2 - 3) >> 3;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
\n\
    vxc_uchar16 in0, in1, tmp, dst0, dst1;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    while (coord_out.y < out_height)\n\
    {\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l10_4x8);\n\
        VXC_DP4x8(dst0, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l11_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize4xUp_l00_4x8);\n\
        VXC_DP4x8(dst1, in1, in0, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize4xUp_l01_4x8);\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.y++;\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
            VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
        coord_in.y += 2;\n\
        coord_out.y++;\n\
    }\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize3xUp_l00_2x8;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l01_2x8;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l10_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l11_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l12_4x4;\n\
_viv_uniform VXC_512Bits uniResize3xUp_l13_4x4;\n\
__kernel void resize_bilinear_U8toU8_SAME_3x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    coord_in.x = (short)(coord_out.x * 2 - 1) / (short)6;\n\
    coord_in.x  = coord_out.x == 0 ? -1 : coord_in.x;\n\
    coord_in.y = (short)(coord_out.y * 2 - 1) / (short)6;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, tmp, dst0, dst1, dst2;\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, in0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in2, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 2),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, in3, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 3),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst2, in1, in0, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l00_2x8);\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(8, 14, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l01_2x8);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst2, in2, in1, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst2,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l00_2x8);\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(8, 14, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l01_2x8);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l10_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(4, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize3xUp_l11_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(8, 11, 0, VXC_RM_ToNearestEven, 1),  uniResize3xUp_l12_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(12, 14, 0, VXC_RM_ToNearestEven, 1), uniResize3xUp_l13_4x4);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst0,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst1,\n\
        VXC_MODIFIER(0, 14, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_bilinear_U8_half_pixel_centers_vx*/

static const char resize_bilinear_U8_opt_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#if (VX_VERSION==2)\n\
\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform int depth;\n\
_viv_uniform VXC_512Bits uniConvertI32toI16_2x8;\n\
_viv_uniform VXC_512Bits uniGetMaskShift_2x8;\n\
_viv_uniform VXC_512Bits uniBilinear_4x4_b;\n\
_viv_uniform float half_pixel_value;\n\
\n\
__kernel void resize_bilinear_U8toU8_UP_opt\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers,\n\
    __read_only  image2d_array_t   scale\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4   coord_x     = coord_out.xxxx + (int4)(0, 1, 2, 3);\n\
    float4 in_x        = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx - half_pixel_value;\n\
\n\
    float4 left_x_f    = floor(in_x);\n\
    int4   left_x_idx  = convert_int4(left_x_f);\n\
    int4   right_x_idx = left_x_idx + 1;\n\
\n\
    float  in_y        = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y - half_pixel_value;\n\
\n\
    float  top_y_f     = floor(in_y);\n\
    int    top_y_idx   = convert_int(top_y_f);\n\
\n\
    vxc_uchar16 src0, src1;\n\
\n\
    vxc_uchar16 top_bottom;\n\
\n\
    int4 coord_in = (int4)(left_x_idx.x, top_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 1),\n\
            VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 bitextract_p0;\n\
    vxc_uchar16 maskShift = {8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};\n\
    VXC_DP2x8(bitextract_p0, left_x_idx, right_x_idx,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvertI32toI16_2x8);\n\
    vxc_ushort8 constData = 8;\n\
    VXC_DP2x8(maskShift, bitextract_p0, constData,\n\
              VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniGetMaskShift_2x8);\n\
\n\
    vxc_ushort16 lerp_0;\n\
    vxc_half16 lerp;\n\
\n\
    int2 coord = (int2)(coord_out.x * 4, coord_out.y);\n\
    VXC_ReadImage(lerp_0.hi, scale, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(lerp_0.lo, scale, coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, lerp.hi, lerp_0.hi, 16);\n\
    _viv_asm(COPY, lerp.lo, lerp_0.lo, 16);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
\n\
    int loop = depth - 1;\n\
    while (coord_in.z < loop)\n\
    {\n\
        VXC_BitExtract(top_bottom, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_BitExtract(top_bottom, src1, src1, maskShift, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        coord_in.zw += (int2)(1, input_desc.s4);\n\
        VXC_OP4(img_load_3d, src0, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
        VXC_OP4(img_load_3d, src1, input, coord_in.xywz,\n\
                VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
        vxc_uchar16 dst;\n\
        VXC_DP4x4_b(dst, lerp.hi, lerp.lo, top_bottom,\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniBilinear_4x4_b);\n\
\n\
        VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
        coord_out.zw += (int2)(1, output_desc.s4);\n\
    }\n\
\n\
    VXC_BitExtract(top_bottom, src0, src0, maskShift, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_BitExtract(top_bottom, src1, src1, maskShift, VXC_MODIFIER(8, 15, 0, VXC_RM_TowardZero, 0));\n\
    vxc_uchar16 dst;\n\
    VXC_DP4x4_b(dst, lerp.hi, lerp.lo, top_bottom,\n\
            VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniBilinear_4x4_b);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
            VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
\n\
}\n\
\n\
#endif"; /* end of resize_bilinear_U8_opt_vx*/

static const char resize_bilinear_nhwc_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniResize_x2_nhwc2_0_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x2_nhwc2_1_4x8;\n\
_viv_uniform int out_height;\n\
\n\
__kernel void resize_bilinear_nhwc_U8toU8_2x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), 0);\n\
    int4 coord_in   = (int4)(get_global_id(0), get_global_id(1), get_global_id(1), 0);\n\
    coord_in.x = ((coord_out.x * 2 - 1) >> 2) - 1;\n\
    coord_in.y = ((coord_out.y * 2 - 1) >> 2);\n\
    coord_in.x  = coord_out.x == 0 ? -2 : coord_in.x;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, result;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in3, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_DP4x8(result, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_DP4x8(result, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in2, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_DP4x8(result, in1, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in1, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.y++;\n\
    VXC_DP4x8(result, in3, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_0_4x8);\n\
    VXC_DP4x8(result, in3, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x2_nhwc2_1_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, result, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l00_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l01_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l02_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l03_2x8;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l10_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l11_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l12_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l13_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l14_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l15_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l16_4x4;\n\
_viv_uniform VXC_512Bits uniResize_x3_nhwc2_l17_4x4;\n\
__kernel void resize_bilinear_nhwc_U8toU8_3x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
    )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(0), get_global_id(1));\n\
    int4 coord_in   = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    coord_in.x = (short)(coord_out.x - 1) / (short)6  * 2;\n\
    coord_in.x  = coord_out.x == 0 ? -2 : coord_in.x;\n\
    coord_in.y = (short)(coord_out.y * 2 - 1) / (short)6;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, dst0, dst1, dst2;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in3, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.zw = coord_out.xy + (int2)(16, 1);\n\
\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in1, in0, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in1, in0, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zy, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l00_2x8);\n\
    VXC_DP2x8(dst0, in1, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l01_2x8);\n\
    VXC_DP2x8(dst1, in1, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l02_2x8);\n\
    VXC_DP2x8(dst1, in1, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l03_2x8);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zw, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.yw += 2;\n\
\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in1, in2, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in1, in2, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zy, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in2, in1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in2, in1, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zw, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
    coord_out.yw += 2;\n\
\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l00_2x8);\n\
    VXC_DP2x8(dst0, in2, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l01_2x8);\n\
    VXC_DP2x8(dst1, in2, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),   uniResize_x3_nhwc2_l02_2x8);\n\
    VXC_DP2x8(dst1, in2, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),  uniResize_x3_nhwc2_l03_2x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zy, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l10_4x4);\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l11_4x4);\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l12_4x4);\n\
    VXC_DP4x4(dst0, in2, in3, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l13_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(0,   3, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l14_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(4,   7, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l15_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(8,  11, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l16_4x4);\n\
    VXC_DP4x4(dst1, in2, in3, VXC_MODIFIER(12, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x3_nhwc2_l17_4x4);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.zw, dst1, VXC_MODIFIER(0, 13, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l00_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l01_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l10_4x8;\n\
_viv_uniform VXC_512Bits uniResize_x4_nhwc2_l11_4x8;\n\
__kernel void resize_bilinear_nhwc_U8toU8_4x_upsample_half_pixel_centers\n\
    (\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
                             int   align_corners,\n\
                             int   half_pixel_centers\n\
     )\n\
{\n\
    int4 coord_out  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(1), get_global_id(1));\n\
    int4 coord_in   = (int4)(get_global_id(0), -1, 0, 0);\n\
    coord_in.x = ((coord_out.x - 3) >> 3) * 2;\n\
    coord_in.y = (coord_out.y * 2 - 3) >> 3;\n\
    coord_in.x  = coord_out.x == 0 ? -2 : coord_in.x;\n\
    coord_in.y  = coord_out.y == 0 ? -1 : coord_in.y;\n\
\n\
    vxc_uchar16 in0, in1, in2, in3, dst0, dst1;\n\
\n\
    VXC_ReadImage(in0, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in1, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 1), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in2, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 2), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    VXC_ReadImage(in3, input, coord_in.xy, VXC_5BITOFFSET_XY(0, 3), VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.zw = coord_out.yy + (int2)(1, 2);\n\
\n\
    VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst0, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst1, in0, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.yz = coord_out.yz + (int2)(3, 3);\n\
\n\
    VXC_DP4x8(dst0, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst0, in2, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_DP4x8(dst1, in2, in1, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst1, in2, in1, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_WriteImage(output, coord_out.xw, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xy, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.yw = coord_out.yw + (int2)(3, 3);\n\
\n\
    VXC_DP4x8(dst0, in1, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst0, in1, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_DP4x8(dst1, in1, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst1, in1, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_WriteImage(output, coord_out.xz, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xw, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
\n\
    coord_out.zw = coord_out.zw + (int2)(3, 3);\n\
\n\
    VXC_DP4x8(dst0, in3, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l10_4x8);\n\
    VXC_DP4x8(dst0, in3, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l11_4x8);\n\
    VXC_DP4x8(dst1, in3, in2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),  uniResize_x4_nhwc2_l00_4x8);\n\
    VXC_DP4x8(dst1, in3, in2, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniResize_x4_nhwc2_l01_4x8);\n\
    VXC_WriteImage(output, coord_out.xy, dst0, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
    VXC_WriteImage(output, coord_out.xz, dst1, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0));\n\
}"; /* end of resize_bilinear_nhwc_vx*/

static const char resize_nearest_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniMultiplyAndPostShift_2x8;\n\
_viv_uniform float2 scale_xy;\n\
_viv_uniform float half_pixel_value;\n\
_viv_uniform float round_value;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4   coord_x    = coord_out.xxxx + (int4)(0, 1, 2, 3); \\\n\
    float4 in_x       = (convert_float4(coord_x) + half_pixel_value) * scale_xy.xxxx + round_value; \\\n\
    int4   in_x_idx   = convert_int4(in_x); \\\n\
    float  in_y       = (convert_float(coord_out.y) + half_pixel_value) * scale_xy.y + round_value; \\\n\
    int    in_y_idx   = convert_int(in_y); \\\n\
\n\
\n\
__kernel void resize_nearest_F16toF16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniGetExtractData_2x8;\n\
__kernel void resize_nearest_F16toF16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniConvertI8toI8_2x8;\n\
__kernel void resize_nearest_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_char16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_I8toI8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst0;\n\
    vxc_char16 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_U8toU8\n\
    (\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(src, src, multiplier, \\\n\
    VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_U8toU8_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_uchar16 src0, dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst, src0, src0, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    vxc_ushort8 multiplier;\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16);\n\
    VXC_DP2x8(dst, dst, multiplier, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniMultiplyAndPostShift_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_I16toI16\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_short8 src;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.y;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(1, 1, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.z;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(2, 2, 0, VXC_RM_TowardZero, 0));\n\
    coord_in.x = in_x_idx.w;\n\
    VXC_OP4(img_load_3d, src, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(3, 3, 0, VXC_RM_TowardZero, 0));\n\
\n\
    VXC_DP2x8(src, src, src, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, src,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void resize_nearest_I16toI16_op\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
    int   align_corners,\n\
    int   half_pixel_centers\n\
    )\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
\n\
    vxc_ushort8 src0, src1, dst0;\n\
    vxc_short8 dst;\n\
    int4 coord_in = (int4)(in_x_idx.x, in_y_idx, coord_out.z, 0);\n\
\n\
    int8 input_desc;\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc));\n\
    int baseAddr = (int)coord_in.z * input_desc.s4 + input_desc.s0;\n\
    _viv_asm(MOV, coord_in.w, baseAddr);\n\
\n\
    VXC_OP4(img_load_3d, src0, input, coord_in.xywz, VXC_5BITOFFSET_XY(0, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    VXC_OP4(img_load_3d, src1, input, coord_in.xywz, VXC_5BITOFFSET_XY(8, 0),\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
    //in_x_idx = in_x_idx - in_x_idx.xxxx;\n\
    vxc_uchar16 mask = (vxc_uchar16)(8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16);\n\
    vxc_ushort8 input_idx;\n\
    _viv_asm(COPY, input_idx, in_x_idx, 16);\n\
    VXC_DP2x8(mask, input_idx, input_idx, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniGetExtractData_2x8);\n\
    VXC_BitExtract(dst0, src0, src1, mask, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
    _viv_asm(COPY, dst, dst0, 8);\n\
    VXC_DP2x8(dst, dst, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertI8toI8_2x8);\n\
\n\
    int8 output_desc;\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc));\n\
    baseAddr = (int)coord_out.z * output_desc.s4 + output_desc.s0;\n\
    _viv_asm(MOV, coord_out.w, baseAddr);\n\
    VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, dst,\n\
                   VXC_MODIFIER(0, 3, 0,VXC_RM_TowardZero, 0));\n\
}\n\
"; /* end of resize_nearest_vx*/

static const char scatter_nd_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int zeropoint;\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
\n\
__kernel void scatter_nd_F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    image2d_array_t  output,\n\
    int width,\n\
    int area,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            VXC_ReadImage(tmpVal, input1, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            _viv_asm(COPY, src, tmpVal, 16);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    _viv_asm(COPY, tmpVal, sum, 16);\n\
    VXC_WriteImage(output, (int2)(gidx, gidy), tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define SCATTER_ND_QINT(src0_type_name, data_type) \\\n\
__kernel void scatter_nd_##src0_type_name##to##src0_type_name##( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    image2d_array_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0);  \\\n\
    int gidy = get_global_id(1); \\\n\
    int firstFlg = 1; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = read_imagei(input0, (int2)(0, i)); \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type src; \\\n\
            VXC_ReadImage(src, input1, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
            if(firstFlg) \\\n\
            { \\\n\
                firstFlg = 0; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    if(firstFlg) \\\n\
    { \\\n\
        sum = (data_type)(zeropoint, zeropoint, zeropoint, zeropoint, \\\n\
                             zeropoint, zeropoint, zeropoint, zeropoint); \\\n\
    } \\\n\
    VXC_WriteImage(output, (int2)(gidx, gidy), sum, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SCATTER_ND_QINT(U8,  vxc_uchar8)\n\
SCATTER_ND_QINT(I8,  vxc_char8)\n\
SCATTER_ND_QINT(I16, vxc_short8)\n\
"; /* end of scatter_nd_vx*/

static const char scatter_nd_big_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
_viv_uniform int zeropoint;\n\
\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
\n\
__kernel void scatter_nd_F16toF16_big(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image i0_img = create_image_from_image2d(input0, 4);\n\
    __global int* index_ptr = (__global int*)i0_img.ptr;\n\
    Image i1_img = create_image_from_image2d(input1, 2);\n\
    __global short* update_ptr = (__global short*)i1_img.ptr;\n\
    Image o_img = create_image_from_image2d(output, 2);\n\
    __global short* output_ptr = (__global short*)o_img.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = vload4(0, index_ptr + i * coord_dim);\n\
\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            short tmpData = update_ptr[i * update_width + gidx];\n\
            _viv_asm(COPY, src, tmpData, 4);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    short dst;\n\
    _viv_asm(COPY, dst, sum, 4);\n\
    output_ptr[gidy * output_width+ gidx] = dst;\n\
}\n\
\n\
#define SCATTER_ND_QINT_BIG(src0_type_name, data_type, ptr_type) \\\n\
__kernel void scatter_nd_##src0_type_name##to##src0_type_name##_big( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    image2d_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0);  \\\n\
    int gidy = get_global_id(1); \\\n\
    int firstFlg = 1; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    Image i0_img = create_image_from_image2d(input0, 2); \\\n\
    __global int* index_ptr = (__global int*)i0_img.ptr; \\\n\
    Image i1_img = create_image_from_image2d(input1, 2); \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)i1_img.ptr; \\\n\
    Image o_img = create_image_from_image2d(output, 2); \\\n\
    __global ptr_type* output_ptr = (__global ptr_type*)o_img.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + i * coord_dim); \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type src; \\\n\
            ptr_type tmpData = update_ptr[i * update_width + gidx]; \\\n\
            _viv_asm(COPY, src, tmpData, 4); \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
            if(firstFlg) \\\n\
            { \\\n\
                firstFlg = 0; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    ptr_type dst; \\\n\
    if(firstFlg) \\\n\
    { \\\n\
        _viv_asm(COPY, dst, zeropoint, 4); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        _viv_asm(COPY, dst, sum, 4); \\\n\
    } \\\n\
    output_ptr[gidy * output_width+ gidx] = dst; \\\n\
}\n\
SCATTER_ND_QINT_BIG(U8,  vxc_uchar8, uchar)\n\
SCATTER_ND_QINT_BIG(I8,  vxc_char8, char)\n\
SCATTER_ND_QINT_BIG(I16, vxc_short8, short)\n\
"; /* end of scatter_nd_big_vx*/

static const char scatter_nd_update_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_1_Lo_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int offset_idx;\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
_viv_uniform int offsetW;\n\
_viv_uniform int2 multAndoutZP0;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
__kernel void scatter_nd_update_F16F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_array_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        //int4 indice = read_imagei(input1, (int2)(0, i));\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            VXC_ReadImage(tmpVal, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpVal, 16);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    _viv_asm(COPY, tmpVal, sum, 16);\n\
    int2 coord = (int2)(gidx, gidy);\n\
    if(cnt == 0)\n\
    {\n\
        VXC_ReadImage(tmpVal, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    VXC_WriteImage(output, coord, tmpVal, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void scatter_nd_update_F16F16toU8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_array_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        //int4 indice = read_imagei(input1, (int2)(0, i));\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            VXC_ReadImage(tmpVal, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpVal, 16);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    int2 coord = (int2)(gidx, gidy);\n\
    vxc_ushort8 ms0;\n\
    vxc_uchar8 dst;\n\
    if(cnt == 0)\n\
    {\n\
        vxc_half8 src;\n\
        VXC_ReadImage(tmpVal, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
        _viv_asm(COPY, src, tmpVal, 16);\n\
        VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_0_Lo_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    else\n\
    {\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16);\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_1_Lo_2x8);\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}\n\
\n\
#define SCATTER_ND_UPDATE_QINT(src0_type_name, src2_type_name, out_type_name, data_type) \\\n\
__kernel void scatter_nd_update_##src0_type_name##src2_type_name##to##out_type_name##( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    image2d_array_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0);  \\\n\
    int gidy = get_global_id(1); \\\n\
    int cnt = 0; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + offset_idx); \\\n\
        index_ptr += coord_dim; \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type src; \\\n\
            VXC_ReadImage(src, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            cnt++; \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
        } \\\n\
    } \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    vxc_ushort8 ms0; \\\n\
    data_type dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        VXC_ReadImage(sum, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16); \\\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_1_Lo_2x8); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT(U8,  U8,  U8,  vxc_uchar8)\n\
SCATTER_ND_UPDATE_QINT(I8,  I8,  I8,  vxc_char8)\n\
SCATTER_ND_UPDATE_QINT(I16, I16, I16, vxc_short8)\n\
\n\
#define SCATTER_ND_UPDATE_QINT_TO_F16(src0_type, data_type) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##toF16( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    image2d_array_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int cnt = 0; \\\n\
    vxc_short8 sum = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    data_type src; \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + offset_idx); \\\n\
        index_ptr += coord_dim; \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            VXC_ReadImage(src, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            cnt++; \\\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8); \\\n\
        } \\\n\
    } \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    vxc_ushort8 ms0; \\\n\
    vxc_half8 tmpDst; \\\n\
    vxc_short8 dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        VXC_ReadImage(src, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        VXC_DP2x8(tmpDst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, 16); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16); \\\n\
        VXC_DP2x8(tmpDst, sum, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_1_Lo_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, 16); \\\n\
        VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_TO_F16(U8,  vxc_uchar8)\n\
SCATTER_ND_UPDATE_QINT_TO_F16(I8,  vxc_char8)\n\
SCATTER_ND_UPDATE_QINT_TO_F16(I16, vxc_short8)\n\
\n\
__kernel void scatter_nd_update_BF16BF16toBF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_array_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_ushort8 src0, src1, src2;\n\
    float4 srcA, srcB;\n\
    float4 sum0 = (float4)(0);\n\
    float4 sum1 = sum0;\n\
\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        //int4 indice = read_imagei(input1, (int2)(0, i));\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            VXC_ReadImage(src0, input2, (int2)(gidx, i), 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
            cnt++;\n\
            VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                        uniConvBF16toF32_Part0_2x8);\n\
            VXC_DP2x8(src2, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\n\
                         uniConvBF16toF32_Part1_2x8);\n\
            _viv_asm(COPY, srcA, src1, 16);\n\
            _viv_asm(COPY, srcB, src2, 16);\n\
            sum0 += srcA;\n\
            sum1 += srcB;\n\
        }\n\
    }\n\
    int2 coord = (int2)(gidx, gidy);\n\
    if(cnt == 0)\n\
    {\n\
        VXC_ReadImage(src2, input0, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
    else\n\
    {\n\
        _viv_asm(COPY, src0, sum0, 16);\n\
        _viv_asm(COPY, src1, sum1, 16);\n\
        VXC_DP2x8(src2, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8);\n\
        VXC_WriteImage(output, coord, src2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    }\n\
}"; /* end of scatter_nd_update_vx*/

static const char scatter_nd_update_atom_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
_viv_uniform int2 multAndoutZP0;\n\
\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
_viv_uniform int offsetW;\n\
_viv_uniform int offset_idx;\n\
\n\
_viv_uniform float scaleInOut;\n\
_viv_uniform float output_zp;\n\
_viv_uniform int input_zp;\n\
_viv_uniform float input_scale;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
_viv_uniform int count_width;\n\
\n\
#define SCATTER_ND_UPDATE_QINT_PRE(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_##src0_type##_pre( \\\n\
    __read_only image2d_t   input1,  __read_only image2d_t   input2, \\\n\
    image2d_t  output, image2d_t  output_cnt, image2d_t tmp_output, \\\n\
    int width, int area, int vol, int coord_dim ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    Image img2 = create_image_from_image2d(input2, element_size); \\\n\
    Image img3 = create_image_from_image2d(output, 4); \\\n\
    Image img4 = create_image_from_image2d(output_cnt, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    __global ptr_type* update_ptr = (__global ptr_type*)img2.ptr; \\\n\
    __global int* output_ptr = (__global int*)img3.ptr; \\\n\
    __global int* cnt_ptr = (__global int*)img4.ptr; \\\n\
    data_type src; \\\n\
 \\\n\
    int4 indice = vload4(0, index_ptr + gidy * coord_dim + offset_idx); \\\n\
    ptr_type tmpData = update_ptr[gidy * update_width + gidx]; \\\n\
    int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
    int loc = idx * output_width + gidx; \\\n\
    _viv_asm(COPY, src, tmpData, 4); \\\n\
    vxc_int4 data; \\\n\
    short zp = input_zp; \\\n\
    VXC_DP4x4(data, src, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 1), \\\n\
             uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    atomic_add(output_ptr + loc, data.x); \\\n\
    if(gidx == 0) \\\n\
    { \\\n\
        atomic_inc(cnt_ptr + idx); \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_PRE(U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_QINT_PRE(I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_QINT_PRE(I16, vxc_short8, short, 2)\n\
\n\
// input0  ref\n\
// input1  sum\n\
// input2  count\n\
// input3  update\n\
#define SCATTER_ND_UPDATE_QINT_TO_F16_BIG(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##toF16_big( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    __read_only image2d_t   input3, \\\n\
    __read_only image2d_t   input4, \\\n\
    image2d_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    Image img2 = create_image_from_image2d(input2, 4); \\\n\
    Image img3 = create_image_from_image2d(output, 2); \\\n\
 \\\n\
    __global int* cnt_ptr = (__global int*)img2.ptr; \\\n\
    __global short* output_ptr = (__global short*)img3.ptr; \\\n\
    data_type src; \\\n\
 \\\n\
    int cnt = cnt_ptr[gidy]; \\\n\
    int loc = gidy * output_width + gidx; \\\n\
 \\\n\
    vxc_ushort8 ms0; \\\n\
    vxc_half8 tmpDst; \\\n\
    vxc_short8 dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        Image img0 = create_image_from_image2d(input0, element_size); \\\n\
        __global ptr_type* ref_ptr = (__global ptr_type*)img0.ptr; \\\n\
        ptr_type tmpData = ref_ptr[loc]; \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        _viv_asm(COPY, src, tmpData, 4); \\\n\
        VXC_DP2x8(tmpDst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        _viv_asm(COPY, dst, tmpDst, 16); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        Image img1 = create_image_from_image2d(input1, 4); \\\n\
        __global int* sum_ptr = (__global int*)img1.ptr; \\\n\
        int sum = sum_ptr[loc]; \\\n\
        float result = sum * input_scale; \\\n\
        half tmpOut; \\\n\
        _viv_asm(CONV, tmpOut, result); \\\n\
        _viv_asm(COPY, dst, tmpOut, 4); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_TO_F16_BIG(U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_QINT_TO_F16_BIG(I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_QINT_TO_F16_BIG(I16, vxc_short8, short, 2)\n\
\n\
#define SCATTER_ND_UPDATE_QINT_BIG(src0_type, data_type, ptr_type, element_size) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##to##src0_type##_big( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    __read_only image2d_t   input3, \\\n\
    __read_only image2d_t   input4, \\\n\
    image2d_t  output, \\\n\
    int width, \\\n\
    int area, \\\n\
    int vol, \\\n\
    int coord_dim \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    Image img2 = create_image_from_image2d(input2, 4); \\\n\
    Image img3 = create_image_from_image2d(output, element_size); \\\n\
    __global int* cnt_ptr = (__global int*)img2.ptr; \\\n\
    __global ptr_type* output_ptr = (__global ptr_type*)img3.ptr; \\\n\
    int cnt = cnt_ptr[gidy]; \\\n\
    int loc = gidy * output_width + gidx; \\\n\
    data_type src, dst; \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        Image img0 = create_image_from_image2d(input0, element_size); \\\n\
        __global ptr_type* ref_ptr = (__global ptr_type*)img0.ptr; \\\n\
        ptr_type tmpData = ref_ptr[loc]; \\\n\
        vxc_ushort8 ms0; \\\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16); \\\n\
        _viv_asm(COPY, src, tmpData, 4); \\\n\
        VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniU8MulAndPostShift_0_Lo_2x8); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
    else \\\n\
    { \\\n\
        Image img1 = create_image_from_image2d(input1, 4); \\\n\
        __global int* sum_ptr = (__global int*)img1.ptr; \\\n\
        int sum = sum_ptr[loc]; \\\n\
        int4 result; \\\n\
        result.x = convert_int_rte(sum * scaleInOut + output_zp); \\\n\
        VXC_DP2x8(dst, result, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), \\\n\
                    uniConvertInt32toUint8_2x8); \\\n\
        output_ptr[loc] = dst.x; \\\n\
    } \\\n\
}\n\
SCATTER_ND_UPDATE_QINT_BIG(U8,  vxc_uchar8, uchar, 1)\n\
SCATTER_ND_UPDATE_QINT_BIG(I8,  vxc_char8,  char,  1)\n\
SCATTER_ND_UPDATE_QINT_BIG(I16, vxc_short8, short, 2)\n\
\n\
__kernel void scatter_nd_update_reset(\n\
    __read_only image2d_t   input0,\n\
    image2d_t  output_sum,\n\
    image2d_t  output_cnt\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
\n\
    Image img3 = create_image_from_image2d(output_sum, 4);\n\
    Image img4 = create_image_from_image2d(output_cnt, 4);\n\
    __global int* sum_ptr = (__global int*)img3.ptr;\n\
    __global int* cnt_ptr = (__global int*)img4.ptr;\n\
    int4 data = (int4)(0);\n\
    vstore4(data, gidx, sum_ptr);\n\
    if(gidx < count_width)\n\
    {\n\
        vstore4(data, gidx, cnt_ptr);\n\
    }\n\
}"; /* end of scatter_nd_update_atom_vx*/

static const char scatter_nd_update_big_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniAccumulateSum_2x8;\n\
_viv_uniform int index_num;\n\
_viv_uniform int update_width;\n\
_viv_uniform int output_width;\n\
\n\
_viv_uniform int offsetX;\n\
_viv_uniform int offsetY;\n\
_viv_uniform int offsetZ;\n\
_viv_uniform int offsetW;\n\
_viv_uniform int offset_idx;\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_0_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_1_Lo_2x8;\n\
_viv_uniform int2 multAndoutZP0;\n\
_viv_uniform int2 multAndoutZP1;//[0:15] multiplier, [31:63] output zp\n\
\n\
__kernel void scatter_nd_update_F16F16toF16_big(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 2);\n\
    Image img3 = create_image_from_image2d(output, 2);\n\
\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global short* update_ptr = (__global short*)img2.ptr;\n\
    __global short* output_ptr = (__global short*)img3.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            short tmpData = update_ptr[i * update_width + gidx];\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpData, 4);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    short dst;\n\
    _viv_asm(COPY, dst, sum, 4);\n\
    int loc = gidy * output_width+ gidx;\n\
    if(cnt == 0)\n\
    {\n\
        Image img0 = create_image_from_image2d(input0, 2);\n\
        __global short* ref_ptr = (__global short*)img0.ptr;\n\
        dst = ref_ptr[loc];\n\
    }\n\
    output_ptr[loc] = dst;\n\
}\n\
\n\
__kernel void scatter_nd_update_F16F16toU8_big(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __read_only image2d_t   input2,\n\
    image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int vol,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int cnt = 0;\n\
\n\
    vxc_short8 tmpVal = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0);\n\
    vxc_half8 sum;\n\
    _viv_asm(COPY, sum, tmpVal, 16);\n\
    Image img1 = create_image_from_image2d(input1, 4);\n\
    Image img2 = create_image_from_image2d(input2, 2);\n\
    Image img3 = create_image_from_image2d(output, 1);\n\
\n\
    __global int* index_ptr = (__global int*)img1.ptr;\n\
    __global short* update_ptr = (__global short*)img2.ptr;\n\
    __global uchar* output_ptr = (__global uchar*)img3.ptr;\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = vload4(0, index_ptr + offset_idx);\n\
        index_ptr += coord_dim;\n\
\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW;\n\
        if(gidy == idx)\n\
        {\n\
            vxc_half8 src;\n\
            short tmpData = update_ptr[i * update_width + gidx];\n\
            cnt++;\n\
            _viv_asm(COPY, src, tmpData, 4);\n\
            VXC_DP2x8(sum, sum, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniAccumulateSum_2x8);\n\
        }\n\
    }\n\
    short dst;\n\
    vxc_ushort8 ms0;\n\
    int loc = gidy * output_width+ gidx;\n\
    if(cnt == 0)\n\
    {\n\
        vxc_half8 src;\n\
        Image img0 = create_image_from_image2d(input0, 2);\n\
        __global short* ref_ptr = (__global short*)img0.ptr;\n\
        short tmpData = ref_ptr[loc];\n\
        _viv_asm(COPY, ms0, multAndoutZP0, 16);\n\
        _viv_asm(COPY, src, tmpData, 4);\n\
        VXC_DP2x8(dst, src, ms0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_0_Lo_2x8);\n\
        output_ptr[loc] = dst;\n\
    }\n\
    else\n\
    {\n\
        _viv_asm(COPY, ms0, multAndoutZP1, 16);\n\
        VXC_DP2x8(dst, sum, ms0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 1),\n\
                    uniU8MulAndPostShift_1_Lo_2x8);\n\
        output_ptr[loc] = dst;\n\
    }\n\
}\n\
"; /* end of scatter_nd_update_big_vx*/

static const char select_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvConditiontoDst_2x8;\n\
_viv_uniform VXC_512Bits uniConvIntIn0toDst_2x8;\n\
_viv_uniform VXC_512Bits uniConvIntIn1toDst_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_PStoF16In0_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_PStoF16In1_2x8;\n\
_viv_uniform int input0Zp;\n\
_viv_uniform int input1Zp;\n\
_viv_uniform int outputZP;\n\
_viv_uniform VXC_512Bits uniU8AddZP_2x8;\n\
\n\
#define SELECT_INT(type_name, read_fun, write_fun) \\\n\
    type_name tmp, src0, src1, dst, value; \\\n\
    vxc_char8 value_tmp; \\\n\
    read_fun(tmp, input0, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src0, tmp, tmp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvIntIn0toDst_2x8); \\\n\
    read_fun(tmp, input1, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(src1, tmp, tmp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvIntIn1toDst_2x8); \\\n\
    read_fun(value_tmp, condition, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(value, value_tmp, value_tmp,\\\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvConditiontoDst_2x8); \\\n\
    dst = (value != 0 ? src0 : src1); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
#define SELECT_INT_FUN(cond_name, src_name, dst_name, type_name) \\\n\
__kernel void select_##cond_name##_##src_name##_##src_name##to##dst_name( \\\n\
    __read_only  image2d_array_t   condition, \\\n\
    __read_only  image2d_array_t   input0, \\\n\
    __read_only  image2d_array_t   input1, \\\n\
    __write_only image2d_array_t   output) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    SELECT_INT(type_name, VXC_ReadImage2DArray, VXC_WriteImage2DArray) \\\n\
}\n\
\n\
SELECT_INT_FUN(I8, I8,  I8,  vxc_char8)\n\
SELECT_INT_FUN(I8, I16, I16, vxc_short8)\n\
\n\
#define SELECT_INT_FUN_2D(cond_name, src_name, dst_name, type_name) \\\n\
__kernel void select_##cond_name##_##src_name##_##src_name##to##dst_name##_2D( \\\n\
    __read_only  image2d_array_t   condition, \\\n\
    __read_only  image2d_array_t   input0, \\\n\
    __read_only  image2d_array_t   input1, \\\n\
    __write_only image2d_array_t   output) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    SELECT_INT(type_name, VXC_ReadImage, VXC_WriteImage) \\\n\
}\n\
\n\
SELECT_INT_FUN_2D(I8, I8,  I8,  vxc_char8)\n\
SELECT_INT_FUN_2D(I8, I16, I16, vxc_short8)\n\
\n\
#define SELECT_HALF(read_fun, write_fun) \\\n\
    vxc_short8 src0, src1, dst, value; \\\n\
    vxc_char8 value_tmp; \\\n\
    read_fun(src0, input0, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(src1, input1, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(value_tmp, condition, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(value, value_tmp, value_tmp,\\\n\
             VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvConditiontoDst_2x8); \\\n\
    dst = (value != 0 ? src0 : src1); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void select_I8_F16_F16toF16(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SELECT_HALF(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void select_I8_F16_F16toF16_2D(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    SELECT_HALF(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define SELECT_U8(read_fun, write_fun) \\\n\
    vxc_uchar8 tmp, src0, src1, dst; \\\n\
    vxc_char8 value; \\\n\
    vxc_half8 tmp1; \\\n\
    vxc_uchar16 input0_ZP, input1_ZP, output_ZP; \\\n\
    _viv_asm(COPY, input0_ZP, input0Zp, 4); \\\n\
    _viv_asm(COPY, input1_ZP, input1Zp, 4); \\\n\
    _viv_asm(COPY, output_ZP, outputZP, 4); \\\n\
    read_fun(tmp, input0, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(tmp1, tmp, input0_ZP, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
             uniU8SubZP_MulM_PStoF16In0_2x8); \\\n\
    VXC_DP2x8(src0, tmp1, output_ZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8AddZP_2x8); \\\n\
    read_fun(tmp, input1, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP2x8(tmp1, tmp, input1_ZP, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
             uniU8SubZP_MulM_PStoF16In1_2x8); \\\n\
    VXC_DP2x8(src1, tmp1, output_ZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8AddZP_2x8); \\\n\
    read_fun(value, condition, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dst = (value != 0 ? src0 : src1); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void select_I8_U8_U8toU8(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SELECT_U8(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void select_I8_U8_U8toU8_2D(\n\
    __read_only  image2d_array_t   condition,\n\
    __read_only  image2d_array_t   input0,\n\
    __read_only  image2d_array_t   input1,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    SELECT_U8(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of select_vx*/

static const char sequence_mask_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits UniFP16toFP32Lo4_dp4x4;\n\
_viv_uniform VXC_512Bits uniConvert1stUint8SubZpToFp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform int inputZP;\n\
_viv_uniform int output_ZP;\n\
_viv_uniform float outputVal1;\n\
\n\
#define SEQUENCE_MASK_QINT_TO_QINT_2D(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void sequence_mask_##src0_type_name##to##src1_type_name##_2D( \\\n\
    image2d_t input, image2d_t output, int maxLen) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int2 coord = (int2)(gidx, get_global_id(1)); \\\n\
    read_type src0; \\\n\
    VXC_ReadImage(src0, input, coord.yy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3); \\\n\
    float4 tmpData; \\\n\
    short zp = inputZP; \\\n\
    VXC_DP4x4(tmpData, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                 uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    int index = convert_int_rte(tmpData.s0 * input_scale); \\\n\
    int4 data; \\\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP; \\\n\
    write_type dst; \\\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I8,  I8,  vxc_char16,  vxc_char16)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I16, I16, vxc_short8,  vxc_short8)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT_2D(I16, U8,  vxc_short8,  vxc_uchar16)\n\
\n\
#define SEQUENCE_MASK_QINT_TO_QINT(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void sequence_mask_##src0_type_name##to##src1_type_name( \\\n\
    image2d_t input, image2d_array_t output, int maxLen) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0); \\\n\
    read_type src0; \\\n\
    VXC_ReadImage(src0, input, coord.yz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3); \\\n\
    float4 tmpData; \\\n\
    short zp = inputZP; \\\n\
    VXC_DP4x4(tmpData, src0, zp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
                 uniConvert1stUint8SubZpToFp32_4x4); \\\n\
    int index = convert_int_rte(tmpData.s0 * input_scale); \\\n\
    int4 data; \\\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP; \\\n\
    write_type dst; \\\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, data, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SEQUENCE_MASK_QINT_TO_QINT(U8,  U8,  vxc_uchar16, vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT(I8,  I8,  vxc_char16,  vxc_char16)\n\
SEQUENCE_MASK_QINT_TO_QINT(I16, I16, vxc_short8,  vxc_short8)\n\
SEQUENCE_MASK_QINT_TO_QINT(I16, U8,  vxc_short8,  vxc_uchar16)\n\
SEQUENCE_MASK_QINT_TO_QINT(I8,  U8,  vxc_char16,  vxc_uchar16)\n\
\n\
__kernel void sequence_mask_F16toF16_2D(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rte(tmpData.x);\n\
    float4 data;\n\
    data = outIdx < index? outputVal1 : convert_float(output_ZP);\n\
    vxc_short8 dst;\n\
    half4 tmpVal;\n\
    _viv_asm(CONV, tmpVal, data);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage(output, coord, dst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void sequence_mask_F16toF16(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rte(tmpData.x);\n\
    float4 data;\n\
    data = outIdx < index? outputVal1 : convert_float(output_ZP);\n\
    vxc_short8 dst;\n\
    half4 tmpVal;\n\
    _viv_asm(CONV, tmpVal, data);\n\
    _viv_asm(COPY, dst, tmpVal, 16);\n\
    VXC_WriteImage2DArray(output, coord, dst.s0246, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void sequence_mask_F16toU8_2D(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yy, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rte(tmpData.x);\n\
    int4 data;\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP;\n\
    vxc_uchar16 dst;\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void sequence_mask_F16toU8(\n\
    image2d_t input, image2d_t output, int maxLen)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    vxc_half8 in_h;\n\
    VXC_ReadImage(src0, input, coord.yz, 0, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
    int4 outIdx = (int4)(gidx, gidx + 1, gidx + 2, gidx + 3);\n\
    _viv_asm(COPY, in_h, src0, 16);\n\
    float4 tmpData;\n\
    VXC_DP4x4(tmpData, in_h, in_h, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
            UniFP16toFP32Lo4_dp4x4);\n\
    int index = convert_int_rte(tmpData.x);\n\
    int4 data;\n\
    data = outIdx < index? convert_int_rte(outputVal1) : output_ZP;\n\
    vxc_uchar16 dst;\n\
    VXC_DP2x8(dst, data, data, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniConvertInt32toUint8_2x8);\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
"; /* end of sequence_mask_vx*/

static const char signal_frame_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define SIGNAL_FRAME_8BITS_SH_IMPL(type) \\\n\
__kernel void signal_frame_##type##to##type \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             frame_step \\\n\
    ) \\\n\
{ \\\n\
    int inner = get_global_id(0); \\\n\
    int length_k = get_global_id(1); \\\n\
    int frames_id = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(inner, length_k, frames_id, frames_id); \\\n\
    int2 coord_in = (int2)(inner, frames_id * frame_step + length_k); \\\n\
 \\\n\
    vxc_uchar16 src; \\\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SIGNAL_FRAME_8BITS_SH_IMPL(U8)\n\
SIGNAL_FRAME_8BITS_SH_IMPL(I8)\n\
\n\
#define SIGNAL_FRAME_16BITS_SH_IMPL(type) \\\n\
__kernel void signal_frame_##type##to##type \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             frame_step \\\n\
    ) \\\n\
{ \\\n\
    int inner = get_global_id(0); \\\n\
    int length_k = get_global_id(1); \\\n\
    int frames_id = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(inner, length_k, frames_id, frames_id); \\\n\
    int2 coord_in = (int2)(inner, frames_id * frame_step + length_k); \\\n\
 \\\n\
    vxc_short8 src; \\\n\
    VXC_ReadImage(src, input, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SIGNAL_FRAME_16BITS_SH_IMPL(I16)\n\
SIGNAL_FRAME_16BITS_SH_IMPL(F16)\n\
SIGNAL_FRAME_16BITS_SH_IMPL(BF16)"; /* end of signal_frame_vx*/

static const char slice_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
#define SLICE_SAMLEFL_SH_IMPL(name, data_type, end_bin) \\\n\
__kernel void slice_##name##_I32to##name##_SAMEFL \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    data_type src; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage2DArray(output, coord, src, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_SAMLEFL_SH_IMPL(U8, vxc_uchar16, 15)\n\
SLICE_SAMLEFL_SH_IMPL(I16, vxc_short8, 7)\n\
\n\
\n\
#define SLICE_SAMLEFL_2D_SH_IMPL(name, data_type, end_bin) \\\n\
__kernel void slice_##name##_I32to##name##_SAMEFL_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    data_type src; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, end_bin, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_SAMLEFL_2D_SH_IMPL(U8, vxc_uchar16, 15)\n\
SLICE_SAMLEFL_2D_SH_IMPL(I16, vxc_short8, 7)\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Lo_2x8;\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Hi_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
#define SLICE_8BITSTO16BITS(name0, name1, src_type, dst_type, save_type) \\\n\
__kernel void slice_##name0##_I32to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    dst_type dst0; \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst0, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO16BITS(I8, F16, vxc_char16,  vxc_half8,  vxc_short8)\n\
SLICE_8BITSTO16BITS(U8, F16, vxc_uchar16, vxc_half8,  vxc_short8)\n\
\n\
#define SLICE_8BITSTO16BITS_2D(name0, name1, src_type, dst_type, save_type) \\\n\
__kernel void slice_##name0##_I32to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src; \\\n\
    dst_type dst0; \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst0, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    save_type dst; \\\n\
    _viv_asm(COPY, dst, dst0, 16); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO16BITS_2D(I8, F16, vxc_char16,  vxc_half8,  vxc_short8)\n\
SLICE_8BITSTO16BITS_2D(U8, F16, vxc_uchar16, vxc_half8,  vxc_short8)\n\
\n\
#define SLICE_8BITSTO8BITS(name0, name1, src_type, dst_type) \\\n\
__kernel void slice_##name0##_I32to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage2DArray(src, input0, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Hi_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO8BITS(I8, I8, vxc_char16,  vxc_char16)\n\
SLICE_8BITSTO8BITS(U8, U8, vxc_uchar16, vxc_uchar16)\n\
\n\
#define SLICE_8BITSTO8BITS_2D(name0, name1, src_type, dst_type) \\\n\
__kernel void slice_##name0##_I32to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src; \\\n\
    dst_type dst; \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage(src, input0, coord_in, 0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Hi_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_8BITSTO8BITS_2D(I8, I8, vxc_char16,  vxc_char16)\n\
SLICE_8BITSTO8BITS_2D(U8, U8, vxc_uchar16, vxc_uchar16)\n\
\n\
#define SLICE_16BITS_TO(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void slice_##name0##_I32to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src; \\\n\
    copy_type src0; \\\n\
    dst_type dst; \\\n\
    int4 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int4 begin = ((int4 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage2DArray(src0, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, src0, 16); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_16BITS_TO(F16, I8,  vxc_half8,  vxc_short8, vxc_char16)\n\
SLICE_16BITS_TO(F16, U8,  vxc_half8,  vxc_short8, vxc_uchar16)\n\
SLICE_16BITS_TO(F16, I16, vxc_half8,  vxc_short8, vxc_short8)\n\
\n\
#define SLICE_16BITS_TO_2D(name0, name1, src_type, copy_type, dst_type) \\\n\
__kernel void slice_##name0##_I32to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_t       input1, \\\n\
    __write_only image2d_array_t output, \\\n\
    int is_samefl \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src; \\\n\
    copy_type src0; \\\n\
    dst_type dst; \\\n\
    int2 coord_in; \\\n\
    Image begin_img = create_image_from_image2d(input1, 4); \\\n\
    uchar* begin_ptr = begin_img.ptr; \\\n\
    int2 begin = ((int2 *)begin_ptr)[0]; \\\n\
    \\\n\
    coord_in = coord + begin; \\\n\
    VXC_ReadImage(src0, input0, coord_in, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src, src0, 16); \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst, src, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniU8MulAndPostShift_Lo_2x8); \\\n\
    VXC_WriteImage(output, coord.xy, dst, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
SLICE_16BITS_TO_2D(F16, I8,  vxc_half8,  vxc_short8, vxc_char16)\n\
SLICE_16BITS_TO_2D(F16, U8,  vxc_half8,  vxc_short8, vxc_uchar16)\n\
SLICE_16BITS_TO_2D(F16, I16, vxc_half8,  vxc_short8, vxc_short8)"; /* end of slice_vx*/

static const char space2depth_internal_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniExtractEvenUint8Stride2_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddUint8Stride2_2x8;\n\
_viv_uniform VXC_512Bits uniExtractEvenFp16Stride2_4x4;\n\
_viv_uniform VXC_512Bits uniExtractOddFp16Stride2_4x4;\n\
\n\
_viv_uniform int input_depth;\n\
\n\
#define SPACE2DEPTH_INTERNAL_QINT_TO_QINT(src0_type_name, src1_type_name, read_type) \\\n\
__kernel void space2depth_internal_##src0_type_name##to##src1_type_name( \\\n\
    image2d_array_t input, \\\n\
    image2d_array_t output, \\\n\
    int block_size_x, \\\n\
    int block_size_y \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    read_type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                    VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    ushort stride_x = (ushort)block_size_x; \\\n\
    ushort stride_y = (ushort)block_size_y; \\\n\
    ushort sidx = (ushort)gidx; \\\n\
    ushort sidy = (ushort)gidy; \\\n\
    ushort tmpX = sidx % stride_x; \\\n\
    ushort tmpY = sidy % stride_y; \\\n\
    int tmpId0 = tmpX; \\\n\
    int tmpId1 = tmpY; \\\n\
    int4 coord_out = (int4)((int)(sidx / stride_x), (int)(sidy / stride_y), 0, 0); \\\n\
    coord_out.z = tmpId0 * input_depth + tmpId1 * block_size_x * input_depth + gidz; \\\n\
    VXC_WriteImage2DArray(output, coord_out, src, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT(U8, U8, vxc_uchar16)\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT(I8, I8, vxc_char16)\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT(I16, I16, vxc_short8)\n\
\n\
__kernel void space2depth_internal_F16toF16(\n\
    image2d_array_t input,\n\
    image2d_array_t output,\n\
    int block_size_x,\n\
    int block_size_y\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0);\n\
    vxc_short8 data, imgVal0;\n\
    VXC_ReadImage2DArray(data, input, coord, VXC_5BITOFFSET_XY(0, 0),\n\
                    VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
\n\
    ushort stride_x = (ushort)block_size_x;\n\
    ushort stride_y = (ushort)block_size_y;\n\
    ushort sidx = (ushort)gidx;\n\
    ushort sidy = (ushort)gidy;\n\
    ushort tmpX = sidx % stride_x;\n\
    ushort tmpY = sidy % stride_y;\n\
    int tmpId0 = tmpX;\n\
    int tmpId1 = tmpY;\n\
    int4 coord_out = (int4)((int)(sidx / stride_x), (int)(sidy / stride_y), 0, 0);\n\
    coord_out.z = tmpId0 * input_depth + tmpId1 * block_size_x * input_depth + gidz;\n\
\n\
    VXC_WriteImage2DArray(output, coord_out, data, VXC_MODIFIER(0, 0, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
#define SPACE2DEPTH_INTERNAL_QINT_TO_QINT_X2Y1(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void space2depth_internal_##src0_type_name##to##src1_type_name##_X2Y1( \\\n\
    image2d_array_t input, \\\n\
    image2d_array_t output, \\\n\
    int block_size_x, \\\n\
    int block_size_y \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_out = (int4)(gidx >> 1, gidy, gidz, 0); \\\n\
    int out_d1; \\\n\
    read_type imageData; \\\n\
    write_type  imgVal0, imgVal1; \\\n\
 \\\n\
    VXC_ReadImage2DArray(imageData, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    out_d1 = gidz + input_depth; \\\n\
 \\\n\
    VXC_DP2x8(imgVal0, imageData, imageData,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractEvenUint8Stride2_2x8); \\\n\
    VXC_DP2x8(imgVal1, imageData, imageData,\\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddUint8Stride2_2x8); \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_out.z = out_d1; \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT_X2Y1(U8, U8, vxc_uchar16, vxc_uchar16)\n\
SPACE2DEPTH_INTERNAL_QINT_TO_QINT_X2Y1(I8, I8, vxc_char16, vxc_char16)\n\
\n\
#define SPACE2DEPTH_INTERNAL_16BITS_X2Y1(src0_type_name, src1_type_name, read_type, write_type) \\\n\
__kernel void space2depth_internal_##src0_type_name##to##src1_type_name##_X2Y1( \\\n\
    image2d_array_t input, \\\n\
    image2d_array_t output, \\\n\
    int block_size_x, \\\n\
    int block_size_y \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int gidz = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, gidy, gidz, 0); \\\n\
    int4 coord_out = (int4)(gidx >> 1, gidy, gidz, 0); \\\n\
    int out_d1; \\\n\
    read_type imageData; \\\n\
    write_type  imgVal0, imgVal1; \\\n\
 \\\n\
    VXC_ReadImage2DArray(imageData, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                     VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    out_d1 = gidz + input_depth; \\\n\
    VXC_DP4x4(imgVal0, imageData, imageData, \\\n\
                 VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractEvenFp16Stride2_4x4); \\\n\
    VXC_DP4x4(imgVal1, imageData, imageData, \\\n\
                VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniExtractOddFp16Stride2_4x4); \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal0, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_out.z = out_d1; \\\n\
    VXC_WriteImage2DArray(output, coord_out, imgVal1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
}\n\
SPACE2DEPTH_INTERNAL_16BITS_X2Y1(I16, I16, vxc_short8, vxc_short8)\n\
SPACE2DEPTH_INTERNAL_16BITS_X2Y1(F16, F16, vxc_short8, vxc_short8)"; /* end of space2depth_internal_vx*/

static const char swish_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform float logE;\n\
\n\
float4 sigmoid_(float4 x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
\n\
_viv_uniform float inputScale;\n\
_viv_uniform float inputTail;\n\
_viv_uniform float outputScale;\n\
_viv_uniform float outputZP;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part0_4x4;\n\
_viv_uniform VXC_512Bits uniDatatoFp32Part1_4x4;\n\
\n\
#define SWISH_PROCESS(read_fun, write_fun, src_type, src_copy_type, convert_type, dst_type, dst_copy_type, \\\n\
                     INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    src_type      src0; \\\n\
    src_copy_type src1; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src1, src0, 16); \\\n\
    float4 vecA, vecB, vecC, vecD; \\\n\
    VXC_DP4x4(vecA, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part0_4x4); \\\n\
    VXC_DP4x4(vecB, src1, src1, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniDatatoFp32Part1_4x4); \\\n\
    vecA = vecA * INSCALE + INTAIL; \\\n\
    vecB = vecB * INSCALE + INTAIL; \\\n\
    vecC = beta * vecA; \\\n\
    vecD = beta * vecB; \\\n\
    vecC = sigmoid_(vecC); \\\n\
    vecD = sigmoid_(vecD); \\\n\
    vecA = vecA * vecC; \\\n\
    vecB = vecB * vecD; \\\n\
    vecA = vecA * OUTSCALE + OUTZP; \\\n\
    vecB = vecB * OUTSCALE + OUTZP; \\\n\
    convert_type dst0, dst1; \\\n\
    _viv_asm(CONV_RTE, dst0, vecA); \\\n\
    _viv_asm(CONV_RTE, dst1, vecB); \\\n\
    dst_type dst2; \\\n\
    VXC_DP2x8(dst2, dst0, dst1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst2, 16); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
#define SWISH_FUNC(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, dst_copy_type,\\\n\
                   INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void swish_##src_type_name##to##dst_type_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    SWISH_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray, src_type, src_copy_type, convert_type, \\\n\
                  dst_type, dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
SWISH_FUNC(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
SWISH_FUNC(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
SWISH_FUNC(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
SWISH_FUNC(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
SWISH_FUNC(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
SWISH_FUNC(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
SWISH_FUNC(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, \\\n\
inputScale, inputTail, outputScale, outputZP)\n\
SWISH_FUNC(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
SWISH_FUNC(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
SWISH_FUNC(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
#define SWISH_FUNC_2D(src_type_name, dst_type_name, src_type, src_copy_type, convert_type, dst_type, \\\n\
                     dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
    __kernel void swish_##src_type_name##to##dst_type_name##_2D( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                           float  beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    SWISH_PROCESS(VXC_ReadImage, VXC_WriteImage, src_type, src_copy_type, convert_type, dst_type, \\\n\
                  dst_copy_type, INSCALE, INTAIL, OUTSCALE, OUTZP) \\\n\
}\n\
\n\
SWISH_FUNC_2D(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8, 1, 0, 1, 0)\n\
SWISH_FUNC_2D(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8,  1, 0, outputScale, 0)\n\
SWISH_FUNC_2D(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8, 1, 0, outputScale, outputZP)\n\
SWISH_FUNC_2D(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8, 1, 0, outputScale, 0)\n\
SWISH_FUNC_2D(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8,  inputScale, 0, outputScale, 0)\n\
SWISH_FUNC_2D(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
SWISH_FUNC_2D(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8, \\\n\
inputScale, inputTail, outputScale, outputZP)\n\
SWISH_FUNC_2D(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8, inputScale, inputTail, 1, 0)\n\
SWISH_FUNC_2D(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8, inputScale, 0, outputScale, 0)\n\
SWISH_FUNC_2D(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8, inputScale, 0, 1, 0)\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part0_2x8;\n\
_viv_uniform VXC_512Bits uniConvBF16toF32_Part1_2x8;\n\
_viv_uniform VXC_512Bits uniExtractOddData_2x8;\n\
\n\
#define SWISH_BF16_PROCESS(read_fun, write_fun) \\\n\
    vxc_ushort8   src0, src1, dst; \\\n\
    float4 vecA, vecB, vecC, vecD; \\\n\
    read_fun(src0, input, coord, 0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 zero = (vxc_short8)(0, 0, 0, 0, 0, 0, 0, 0); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part0_2x8); \\\n\
    _viv_asm(COPY, vecA, src1, 16); \\\n\
    VXC_DP2x8(src1, src0, zero, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniConvBF16toF32_Part1_2x8); \\\n\
    _viv_asm(COPY, vecB, src1, 16); \\\n\
    vecC = beta * vecA; \\\n\
    vecD = beta * vecB; \\\n\
    vecC = sigmoid_(vecC); \\\n\
    vecD = sigmoid_(vecD); \\\n\
    vecA = vecA * vecC; \\\n\
    vecB = vecB * vecD; \\\n\
    _viv_asm(COPY, src0, vecA, 16); \\\n\
    _viv_asm(COPY, src1, vecB, 16); \\\n\
    VXC_DP2x8(dst, src0, src1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), uniExtractOddData_2x8); \\\n\
    write_fun(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void swish_BF16toBF16(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_BF16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray);\n\
}\n\
\n\
__kernel void swish_BF16toBF16_2D(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  beta\n\
    )\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_BF16_PROCESS(VXC_ReadImage, VXC_WriteImage);\n\
}\n\
"; /* end of swish_vx*/

static const char tensorstackconcat_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
__kernel void tensorstackconcat_16bits\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.w = 0;\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void tensorstackconcat_8bits\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int idx = coord.x;\n\
    vxc_char16 src0, src1;\n\
    VXC_ReadImage2DArray(src0, input, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage2DArray(output, coord, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void tensorstackconcat_16bits_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    vxc_short8 src0;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage(output, coord.xy, src0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
}\n\
\n\
__kernel void tensorstackconcat_8bits_2D\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __read_only  image2d_t       index,\n\
    __write_only image2d_array_t output\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int idx = coord.x;\n\
    vxc_char16 src0, src1;\n\
    VXC_ReadImage(src0, input, coord.xy, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
    coord.y = read_imagei(index, coord.ww).x;\n\
    VXC_WriteImage(output, coord.xy, src0, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
}"; /* end of tensorstackconcat_vx*/

static const char tile_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int lastWorkItem;\n\
\n\
#define TILE_3D(name0, name1, name2, remainder, type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out; \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    type src; \\\n\
    VXC_ReadImage2DArray(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
 \\\n\
    int batch_id = (short)coord.z / (short)depthIn; \\\n\
    coord.z = (short)coord.z % (short)depthIn; \\\n\
    coord_out = coord; \\\n\
 \\\n\
    for (int w = 0; w < multiples_3; w++) \\\n\
    { \\\n\
        int batch = batchIn * w + batch_id; \\\n\
 \\\n\
        for(int z = 0; z < multiples_2; z++) \\\n\
        { \\\n\
            coord_out.z = coord.z + z * depthIn + batch * depthOut; \\\n\
 \\\n\
            for (int y = 0; y < multiples_1; y++) \\\n\
            { \\\n\
                coord_out.y = coord.y + y * height; \\\n\
 \\\n\
                for (int x = 0; x < multiples_0; x++) \\\n\
                { \\\n\
                    coord_out.x = coord.x + x * width; \\\n\
                    if (isLastItem) \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, src, \\\n\
                            VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                    else \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, src, \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
}\n\
TILE_3D(U8, U8, 1, 0, vxc_uchar8)\n\
TILE_3D(U8, U8, 2, 1, vxc_uchar8)\n\
TILE_3D(U8, U8, 3, 2, vxc_uchar8)\n\
TILE_3D(U8, U8, 4, 3, vxc_uchar8)\n\
TILE_3D(U8, U8, 5, 4, vxc_uchar8)\n\
TILE_3D(U8, U8, 6, 5, vxc_uchar8)\n\
TILE_3D(U8, U8, 7, 6, vxc_uchar8)\n\
TILE_3D(U8, U8, 0, 7, vxc_uchar8)\n\
\n\
TILE_3D(I16, I16, 1, 0, vxc_short8)\n\
TILE_3D(I16, I16, 2, 1, vxc_short8)\n\
TILE_3D(I16, I16, 3, 2, vxc_short8)\n\
TILE_3D(I16, I16, 4, 3, vxc_short8)\n\
TILE_3D(I16, I16, 5, 4, vxc_short8)\n\
TILE_3D(I16, I16, 6, 5, vxc_short8)\n\
TILE_3D(I16, I16, 7, 6, vxc_short8)\n\
TILE_3D(I16, I16, 0, 7, vxc_short8)\n\
\n\
\n\
#define TILE_2D(name0, name1, name2, remainder, type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1##_2D( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
    type src; \\\n\
    VXC_ReadImage(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            if (isLastItem) \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            else \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            coord.x += width; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = get_global_id(0); \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D(U8, U8, 1, 0, vxc_uchar8)\n\
TILE_2D(U8, U8, 2, 1, vxc_uchar8)\n\
TILE_2D(U8, U8, 3, 2, vxc_uchar8)\n\
TILE_2D(U8, U8, 4, 3, vxc_uchar8)\n\
TILE_2D(U8, U8, 5, 4, vxc_uchar8)\n\
TILE_2D(U8, U8, 6, 5, vxc_uchar8)\n\
TILE_2D(U8, U8, 7, 6, vxc_uchar8)\n\
TILE_2D(U8, U8, 0, 7, vxc_uchar8)\n\
\n\
TILE_2D(I16, I16, 1, 0, vxc_short8)\n\
TILE_2D(I16, I16, 2, 1, vxc_short8)\n\
TILE_2D(I16, I16, 3, 2, vxc_short8)\n\
TILE_2D(I16, I16, 4, 3, vxc_short8)\n\
TILE_2D(I16, I16, 5, 4, vxc_short8)\n\
TILE_2D(I16, I16, 6, 5, vxc_short8)\n\
TILE_2D(I16, I16, 7, 6, vxc_short8)\n\
TILE_2D(I16, I16, 0, 7, vxc_short8)\n\
\n\
#define TILE_2D_1TON(name0, name1, type) \\\n\
__kernel void tile_1toN_##name0##to##name1##_2D( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
    type src; \\\n\
    VXC_ReadImage(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            VXC_WriteImage(output, coord, src, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            coord.x += 8; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = 0; \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D_1TON(U8,  U8, vxc_uchar8)\n\
TILE_2D_1TON(I16, I16, vxc_short8)\n\
"; /* end of tile_vx*/

static const char tile_mix_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniU8MulAndPostShift_Lo_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
_viv_uniform int lastWorkItem;\n\
\n\
#define TILE_3D_MIX(name0, name1, name2, remainder, type, out_type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out; \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    type src; \\\n\
    vxc_half8  src1; \\\n\
    out_type dst; \\\n\
    VXC_ReadImage2DArray(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
 \\\n\
    int batch_id = (short)coord.z / (short)depthIn; \\\n\
    coord.z = (short)coord.z % (short)depthIn; \\\n\
    coord_out = coord; \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(src1, src, multiplier,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    _viv_asm(COPY, dst, src1, 16); \\\n\
 \\\n\
    for (int w = 0; w < multiples_3; w++) \\\n\
    { \\\n\
        int batch = batchIn * w + batch_id; \\\n\
 \\\n\
        for(int z = 0; z < multiples_2; z++) \\\n\
        { \\\n\
            coord_out.z = coord.z + z * depthIn + batch * depthOut; \\\n\
 \\\n\
            for (int y = 0; y < multiples_1; y++) \\\n\
            { \\\n\
                coord_out.y = coord.y + y * height; \\\n\
 \\\n\
                for (int x = 0; x < multiples_0; x++) \\\n\
                { \\\n\
                    coord_out.x = coord.x + x * width; \\\n\
                    if (isLastItem) \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, dst, \\\n\
                            VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                    else \\\n\
                    { \\\n\
                        VXC_WriteImage2DArray(output, coord_out, dst, \\\n\
                            VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
                    } \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
}\n\
TILE_3D_MIX(U8, F16, 1, 0, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 2, 1, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 3, 2, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 4, 3, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 5, 4, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 6, 5, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 7, 6, vxc_uchar8, vxc_short8)\n\
TILE_3D_MIX(U8, F16, 0, 7, vxc_uchar8, vxc_short8)\n\
\n\
\n\
#define TILE_2D_MIX(name0, name1, name2, remainder, type, out_type) \\\n\
__kernel void tile_remain##name2##_##name0##to##name1##_2D( \\\n\
    __read_only image2d_array_t  input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
    type src; \\\n\
    vxc_half8  src1; \\\n\
    out_type dst; \\\n\
    VXC_ReadImage(src, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
 \\\n\
    int isLastItem = coord.x == lastWorkItem; \\\n\
 \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(src1, src, multiplier,\\\n\
            VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniU8MulAndPostShift_Lo_2x8); \\\n\
    _viv_asm(COPY, dst, src1, 16); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            if (isLastItem) \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, remainder, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            else \\\n\
            { \\\n\
                VXC_WriteImage(output, coord, dst, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
            } \\\n\
            coord.x += width; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = get_global_id(0); \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D_MIX(U8, F16, 1, 0, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 2, 1, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 3, 2, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 4, 3, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 5, 4, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 6, 5, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 7, 6, vxc_uchar8, vxc_short8)\n\
TILE_2D_MIX(U8, F16, 0, 7, vxc_uchar8, vxc_short8)\n\
"; /* end of tile_mix_vx*/

static const char upsample_F16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniF16MulMultipiler_PostShft_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
#define UPSAMPLE_F16_U8TO_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short8 din0; \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_half8 src; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 axisData1; \\\n\
    vxc_uchar16 dout; \\\n\
    read_fun(din0, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    vxc_short8 tmp; \\\n\
    uchar zp = 0; \\\n\
    _viv_asm(COPY, src, din0, 16); \\\n\
    VXC_DP2x8(tmp, src, src, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniF16MulMultipiler_PostShft_2x8); \\\n\
    vxc_uchar16 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(din, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    constAxis    = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    dinExpand    = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_F16_U8to_U8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_U8TO_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_U8to_U8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_U8TO_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits shortMulShort_8x8;\n\
_viv_uniform VXC_512Bits uniConvertFstFp16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertSecFp16Fp32_4x4;\n\
_viv_uniform int upOutput_ZP;\n\
_viv_uniform float upOutput_Scale;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8;\n\
\n\
\n\
#define UPSAMPLE_F16_I16TO_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_short4 axisIn; \\\n\
    vxc_short8 dinExp, axisInExp, constAxis,axisData,tmpout; \\\n\
    vxc_half8 dout; \\\n\
    vxc_float4 tmpVal1, tmpVal2, convZp; \\\n\
    vxc_int4 tmpData1, tmpData2, tmpData3; \\\n\
    vxc_uchar8 result; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_short8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_short8)(1); \\\n\
    VXC_DP2x8(tmpout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), shortMulShort_8x8); \\\n\
    _viv_asm(COPY, dout, tmpout, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 /= upOutput_Scale; \\\n\
    tmpVal2 /= upOutput_Scale; \\\n\
    tmpData3 = isnotequal(tmpVal1, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal1 += convZp; \\\n\
    tmpData3 = isnotequal(tmpVal2, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal2 += convZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_short8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_short8)(1); \\\n\
    VXC_DP2x8(tmpout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), shortMulShort_8x8); \\\n\
    _viv_asm(COPY, dout, tmpout, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 /= upOutput_Scale; \\\n\
    tmpVal2 /= upOutput_Scale; \\\n\
    tmpData3 = isnotequal(tmpVal1, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal1 += convZp; \\\n\
    tmpData3 = isnotequal(tmpVal2, 0); \\\n\
    tmpData3 *= (-upOutput_ZP); \\\n\
    convZp = convert_float4_rtp(tmpData3); \\\n\
    tmpVal2 += convZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_F16_I16to_U8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_I16TO_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_I16to_U8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_I16TO_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform float scaleOut;\n\
_viv_uniform float outputZp;\n\
_viv_uniform VXC_512Bits ucharMulShort_8x8_2;\n\
\n\
#define UPSAMPLE_F16_U8TO_I8_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_uchar4 axisIn; \\\n\
    vxc_short8 dinExp, tmpOut; \\\n\
    vxc_uchar8 axisInExp; \\\n\
    vxc_uchar8 constAxis; \\\n\
    vxc_uchar8 axisData; \\\n\
    vxc_half8 dout; \\\n\
    vxc_float4 tmpVal0, tmpVal1; \\\n\
    vxc_char8 result; \\\n\
    int4 tmpData1, tmpData2; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_uchar8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal0 = tmpVal0 * scaleOut + outputZp; \\\n\
    tmpVal1 = tmpVal1 * scaleOut + outputZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal0); \\\n\
    tmpData2 = convert_int4_rte(tmpVal1); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    coordOut.y += 1; \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal0 = tmpVal0 * scaleOut + outputZp; \\\n\
    tmpVal1 = tmpVal1 * scaleOut + outputZp; \\\n\
    tmpData1 = convert_int4_rte(tmpVal0); \\\n\
    tmpData2 = convert_int4_rte(tmpVal1); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_F16_U8to_I8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_U8TO_I8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_U8to_I8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_U8TO_I8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform float up_outFlScale_i16;\n\
\n\
#define UPSAMPLE_F16_U8TO_I16_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_uchar4 axisIn; \\\n\
    vxc_short8 dinExp, tmpOut; \\\n\
    vxc_uchar8 axisInExp; \\\n\
    vxc_uchar8 constAxis; \\\n\
    vxc_uchar8 axisData; \\\n\
    half8 dout; \\\n\
    float4 tmpVal1, tmpVal2; \\\n\
    int4 tmpData1, tmpData2; \\\n\
    vxc_short8 result; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_uchar8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 *= up_outFlScale_i16; \\\n\
    tmpVal2 *= up_outFlScale_i16; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(tmpOut, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_8x8_2); \\\n\
    coordOut.y += 1; \\\n\
    _viv_asm(COPY, dout, tmpOut, 16); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertFstFp16Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertSecFp16Fp32_4x4); \\\n\
    tmpVal1 *= up_outFlScale_i16; \\\n\
    tmpVal2 *= up_outFlScale_i16; \\\n\
    tmpData1 = convert_int4_rte(tmpVal1); \\\n\
    tmpData2 = convert_int4_rte(tmpVal2); \\\n\
    VXC_DP2x8(result, tmpData1, tmpData2, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), \\\n\
        uniConvertInt32toUint8_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_F16_U8to_I16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_F16_U8TO_I16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_F16_U8to_I16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_F16_U8TO_I16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_F16_vx*/

static const char upsample_I16_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
//--------------------------unpooling-------------------------\n\
_viv_uniform VXC_512Bits uniQuantInOutInt16_2x8;\n\
_viv_uniform VXC_512Bits ucharMulShort_2x8;\n\
\n\
#define UPSAMPLE_I16_U8TO_I16_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    vxc_uchar4 axisIn; \\\n\
    vxc_short8 dinExp; \\\n\
    vxc_uchar8 axisInExp; \\\n\
    vxc_uchar8 constAxis; \\\n\
    vxc_uchar8 axisData; \\\n\
    vxc_short8 axisData_short; \\\n\
    vxc_short8 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (vxc_uchar8)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar8)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (vxc_uchar8)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_I16_U8to_I16_SAME\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_U8TO_I16_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_U8to_I16_SAME_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_U8TO_I16_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
#define UPSAMPLE_I16_TO_I16_PROCESS(axis_type, axis_in_type, read_fun, write_fun) \\\n\
    vxc_short4 din; \\\n\
    axis_in_type axisIn; \\\n\
    vxc_short8 dinExp; \\\n\
    axis_type  axisInExp; \\\n\
    axis_type  constAxis; \\\n\
    axis_type  axisData; \\\n\
    vxc_short8 axisData_short; \\\n\
    vxc_short8 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExp = din.s00112233; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (axis_type)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    VXC_DP2x8(dout, dout, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantInOutInt16_2x8); \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (axis_type)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(dout, axisData, dinExp, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), ucharMulShort_2x8); \\\n\
    VXC_DP2x8(dout, dout, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniQuantInOutInt16_2x8); \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_I16_U8to_I16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_U8to_I16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
__kernel void upsample_I16_I16to_I16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_I16to_I16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_I16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirInt16Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertU8toI16_2x8;\n\
_viv_uniform float inScaleInt16;\n\
\n\
#define UPSAMPLE_I16_TO_F16_PROCESS(axis_type, axis_in_type, read_fun, write_fun) \\\n\
    vxc_short8 din; \\\n\
    axis_in_type axisIn; \\\n\
    vxc_short8 dinExp; \\\n\
    axis_type  axisInExp; \\\n\
    axis_type  constAxis; \\\n\
    axis_type  axisData; \\\n\
    vxc_short8 axisData_short; \\\n\
    vxc_short8 dout; \\\n\
    vxc_float4 tmpVal0, tmpVal1; \\\n\
    half4 tmpOut0; \\\n\
    vxc_short8 tmpOut1; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_DP4x4(tmpVal0, din, din, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertDirInt16Fp32_4x4); \\\n\
    tmpVal1 = tmpVal0 * inScaleInt16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal1); \\\n\
    _viv_asm(COPY, tmpOut1, tmpOut0, 16); \\\n\
    dinExp    = tmpOut1.s00224466; \\\n\
    axisInExp = axisIn.s00112233; \\\n\
    constAxis = (axis_type)(0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(axisData_short, axisData, axisData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertU8toI16_2x8); \\\n\
    dout = axisData_short == 1 ? dinExp : 0; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (axis_type)(2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExp, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 7, 0, 1)); \\\n\
    axisData &= (axis_type)(1); \\\n\
    VXC_DP2x8(axisData_short, axisData, axisData, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertU8toI16_2x8); \\\n\
    dout = axisData_short == 1 ? dinExp : 0; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
__kernel void upsample_I16_I16to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_I16to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_short8, vxc_short4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
__kernel void upsample_I16_U8to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I16_U8to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I16_TO_F16_PROCESS(vxc_uchar8, vxc_uchar4, VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_I16_vx*/

static const char upsample_I8_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
\n\
#define UPSAMPLE_I8_U8TO_I8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_char8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_char16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_char16 zpValue; \\\n\
    vxc_char16 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    zpValue = (char)input_ZP; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_I8_U8to_I8_SAME\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I8_U8TO_I8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I8_U8to_I8_SAME_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I8_U8TO_I8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform VXC_512Bits uniS16MoveValue_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
#define UPSAMPLE_I8_U8TO_I8_PROCESS(read_fun, write_fun) \\\n\
    vxc_char8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_char16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_char16 zpValue; \\\n\
    vxc_char16 dout; \\\n\
    vxc_char16 result, result_tmp; \\\n\
    zpValue = (char)input_ZP; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    vxc_short8 tmp; \\\n\
    short zp = input_ZP; \\\n\
    vxc_short8 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_I8_U8to_I8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_I8_U8TO_I8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I8_U8to_I8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_I8_U8TO_I8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniConvertDirUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertEndUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertTrdUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertFthUint8Fp32_4x4_2;\n\
_viv_uniform VXC_512Bits uniConvertInt32toUint8_2x8_2;\n\
_viv_uniform float scaleIn;\n\
_viv_uniform float inputTail;\n\
\n\
#define UPSAMPLE_I8_U8TO_F16_PROCESS(read_fun, write_fun) \\\n\
    vxc_char8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_char16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_char16 zpValue; \\\n\
    vxc_char16 dout; \\\n\
    zpValue = (char)input_ZP; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coordOut1.x += 8; \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_short8 rout0, rout1; \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertDirUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertEndUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertTrdUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFthUint8Fp32_4x4_2); \\\n\
    tmpVal0 = tmpVal0 * scaleIn + inputTail; \\\n\
    tmpVal1 = tmpVal1 * scaleIn + inputTail; \\\n\
    tmpVal2 = tmpVal2 * scaleIn + inputTail; \\\n\
    tmpVal3 = tmpVal3 * scaleIn + inputTail; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertDirUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertEndUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertTrdUint8Fp32_4x4_2); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertFthUint8Fp32_4x4_2); \\\n\
    tmpVal0 = tmpVal0 * scaleIn + inputTail; \\\n\
    tmpVal1 = tmpVal1 * scaleIn + inputTail; \\\n\
    tmpVal2 = tmpVal2 * scaleIn + inputTail; \\\n\
    tmpVal3 = tmpVal3 * scaleIn + inputTail; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0), \\\n\
        uniConvertInt32toUint8_2x8_2); \\\n\
    coordOut.y += 1; \\\n\
    coordOut1.y += 1; \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_I8_U8to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    int4 coordOut1 = coordOut;\n\
    UPSAMPLE_I8_U8TO_F16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_I8_U8to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    int2 coordOut1 = coordOut;\n\
    UPSAMPLE_I8_U8TO_F16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_I8_vx*/

static const char upsample_U8_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform int input_ZP;\n\
\n\
#define UPSAMPLE_U8_U8TO_U8_SAME_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 zpValue; \\\n\
    vxc_uchar16 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    zpValue = (uchar)input_ZP; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    write_fun(dataOut, coordOut, dout, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_U8_U8to_U8_SAME\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_U8_U8TO_U8_SAME_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_U8_U8to_U8_SAME_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_U8_U8TO_U8_SAME_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_2x8;\n\
_viv_uniform VXC_512Bits uniU8SubZP_MulM_Hi_2x8;\n\
_viv_uniform VXC_512Bits uniS16AddOutZP_2x8;\n\
_viv_uniform VXC_512Bits uniS16MoveValue_2x8;\n\
_viv_uniform vxc_uint4 packed_outputZP;\n\
\n\
#define UPSAMPLE_U8_U8TO_U8_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 zpValue; \\\n\
    vxc_uchar16 dout; \\\n\
    vxc_uchar16 result, result_tmp; \\\n\
    zpValue = (uchar)input_ZP; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    vxc_short8 tmp; \\\n\
    short zp = input_ZP; \\\n\
    vxc_short8 packed_outZP; \\\n\
    _viv_asm(COPY, packed_outZP, packed_outputZP, 16); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    dout = axisData ? dinExpand : zpValue; \\\n\
    coordOut.y += 1; \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_2x8); \\\n\
    VXC_DP2x8(result, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(tmp, dout, zp, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
              uniU8SubZP_MulM_Hi_2x8); \\\n\
    VXC_DP2x8(result_tmp, tmp, packed_outZP, VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16AddOutZP_2x8); \\\n\
    VXC_DP2x8(result, result_tmp, result_tmp, VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1),\\\n\
        uniS16MoveValue_2x8); \\\n\
    write_fun(dataOut, coordOut, result, VXC_MODIFIER(0, 15, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_U8_U8to_U8\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    UPSAMPLE_U8_U8TO_U8_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_U8_U8to_U8_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord    = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut = (int2)(coord.x << 1, coord.y << 1);\n\
    UPSAMPLE_U8_U8TO_U8_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
\n\
\n\
_viv_uniform VXC_512Bits uniMulMinusZpUint8_4x4;\n\
_viv_uniform VXC_512Bits uniMulMinusZp2Uint8_4x4;\n\
_viv_uniform VXC_512Bits uniMulMinusZp3Uint8_4x4;\n\
_viv_uniform VXC_512Bits uniMulMinusZp4Uint8_4x4;\n\
_viv_uniform VXC_512Bits uniConvertInt32toInt16_2x8;\n\
_viv_uniform VXC_512Bits uniConvertDirUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertEndUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertTrdUint8Fp32_4x4;\n\
_viv_uniform VXC_512Bits uniConvertFthUint8Fp32_4x4;\n\
_viv_uniform float scaleU8Fp16;\n\
_viv_uniform int zpU8Fp16;\n\
\n\
#define UPSAMPLE_U8_U8TO_F16_PROCESS(read_fun, write_fun) \\\n\
    vxc_uchar8 din; \\\n\
    vxc_uchar8 axisIn; \\\n\
    vxc_uchar16 dinExpand; \\\n\
    vxc_uchar16 axisInExpand; \\\n\
    vxc_uchar16 constAxis; \\\n\
    vxc_uchar16 axisData; \\\n\
    vxc_uchar16 axisData1; \\\n\
    vxc_uchar16 dout; \\\n\
    read_fun(din, dataIn, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    read_fun(axisIn, axis, coord, VXC_5BITOFFSET_XY(0, 0),\\\n\
        VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    coordOut1.x += 8; \\\n\
    dinExpand = din.s0011223344556677; \\\n\
    axisInExpand = axisIn.s0011223344556677; \\\n\
    constAxis = (vxc_uchar16)(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    vxc_float4 tmpVal0, tmpVal1, tmpVal2, tmpVal3, convZp; \\\n\
    half4 tmpOut0, tmpOut1; \\\n\
    vxc_short8 rout0, rout1; \\\n\
    vxc_int4 tmpV0, tmpV1, tmpV2, tmpV3; \\\n\
    vxc_float4 tmpData0, tmpData1, tmpData2, tmpData3; \\\n\
    short tmpZp = (short)(-zpU8Fp16); \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertDirUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertTrdUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertFthUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpV0, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZpUint8_4x4); \\\n\
    VXC_DP4x4(tmpV1, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp2Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV2, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp3Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV3, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp4Uint8_4x4); \\\n\
    _viv_asm(CONV, tmpData0, tmpV0); \\\n\
    _viv_asm(CONV, tmpData1, tmpV1); \\\n\
    _viv_asm(CONV, tmpData2, tmpV2); \\\n\
    _viv_asm(CONV, tmpData3, tmpV3); \\\n\
    tmpVal0 = (tmpVal0 + tmpData0) * scaleU8Fp16; \\\n\
    tmpVal1 = (tmpVal1 + tmpData1) * scaleU8Fp16; \\\n\
    tmpVal2 = (tmpVal2 + tmpData2) * scaleU8Fp16; \\\n\
    tmpVal3 = (tmpVal3 + tmpData3) * scaleU8Fp16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    constAxis = (vxc_uchar16)(2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3); \\\n\
    VXC_Clamp(axisData, axisInExpand, constAxis, constAxis, VXC_MODIFIER_CLAMP(0, 15, 0, 1)); \\\n\
    axisData &= (vxc_uchar16)(1); \\\n\
    _viv_asm(COPY, axisData1, axisData, 16); \\\n\
    dout = axisData1 * dinExpand; \\\n\
    VXC_DP4x4(tmpVal0, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertDirUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal1, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertEndUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal2, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertTrdUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpVal3, dout, dout, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertFthUint8Fp32_4x4); \\\n\
    VXC_DP4x4(tmpV0, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZpUint8_4x4); \\\n\
    VXC_DP4x4(tmpV1, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp2Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV2, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp3Uint8_4x4); \\\n\
    VXC_DP4x4(tmpV3, axisData1, tmpZp, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0),\\\n\
        uniMulMinusZp4Uint8_4x4); \\\n\
    _viv_asm(CONV, tmpData0, tmpV0); \\\n\
    _viv_asm(CONV, tmpData1, tmpV1); \\\n\
    _viv_asm(CONV, tmpData2, tmpV2); \\\n\
    _viv_asm(CONV, tmpData3, tmpV3); \\\n\
    tmpVal0 = (tmpVal0 + tmpData0) * scaleU8Fp16; \\\n\
    tmpVal1 = (tmpVal1 + tmpData1) * scaleU8Fp16; \\\n\
    tmpVal2 = (tmpVal2 + tmpData2) * scaleU8Fp16; \\\n\
    tmpVal3 = (tmpVal3 + tmpData3) * scaleU8Fp16; \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal0); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal1); \\\n\
    VXC_DP2x8(rout0, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    _viv_asm(CONV, tmpOut0, tmpVal2); \\\n\
    _viv_asm(CONV, tmpOut1, tmpVal3); \\\n\
    VXC_DP2x8(rout1, tmpOut0, tmpOut1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0),\\\n\
        uniConvertInt32toInt16_2x8); \\\n\
    coordOut.y += 1; \\\n\
    coordOut1.y += 1; \\\n\
    write_fun(dataOut, coordOut, rout0, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    write_fun(dataOut, coordOut1, rout1, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0));\n\
\n\
\n\
__kernel void upsample_U8_U8to_F16\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int4 coord     = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coordOut  = (int4)(coord.x << 1, coord.y << 1, coord.z, 0);\n\
    int4 coordOut1 = coordOut;\n\
    UPSAMPLE_U8_U8TO_F16_PROCESS(VXC_ReadImage2DArray, VXC_WriteImage2DArray)\n\
}\n\
\n\
__kernel void upsample_U8_U8to_F16_2D\n\
    (\n\
        image2d_array_t dataIn,\n\
        image2d_array_t axis,\n\
        image2d_array_t dataOut\n\
    )\n\
{\n\
    int2 coord     = (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coordOut  = (int2)(coord.x << 1, coord.y << 1);\n\
    int2 coordOut1 = coordOut;\n\
    UPSAMPLE_U8_U8TO_F16_PROCESS(VXC_ReadImage, VXC_WriteImage)\n\
}\n\
"; /* end of upsample_U8_vx*/

static const char upsamplescale_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertDatatoF32_4x4;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float tail;\n\
\n\
#define UPSAMPLE_SCALETO_FUN(src_name, dst_name, read_type, src_type, dst_type, write_type, conv_func) \\\n\
    __kernel void upsamplescale_##src_name##to##dst_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride, \\\n\
                 float            scale) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst_val; \\\n\
    write_type write_val; \\\n\
    VXC_ReadImage2DArray(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    coord.xy *= stride; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.w, baseAddr); \\\n\
    float4 data; \\\n\
    VXC_DP4x4(data, src_val, src_val, VXC_MODIFIER(0, 3, 0, VXC_RM_ToNearestEven, 1), uniConvertDatatoF32_4x4); \\\n\
    data = data * output_scale + tail; \\\n\
    _viv_asm(conv_func, dst_val, data); \\\n\
    _viv_asm(COPY, write_val, dst_val, 16); \\\n\
    int4 coord_out = coord; \\\n\
    for (int y = 0; y < stride; y++) \\\n\
    { \\\n\
        coord_out.x = coord.x; \\\n\
        for (int x = 0; x < stride; ) \\\n\
        { \\\n\
            VXC_OP4_NoDest(img_store_3d, output, coord_out.xywz, write_val, \\\n\
                VXC_MODIFIER(0, 0, 0,VXC_RM_TowardZero, 0)); \\\n\
            x++; \\\n\
            coord_out.x ++; \\\n\
        } \\\n\
        coord_out.y ++; \\\n\
    } \\\n\
}\n\
\n\
UPSAMPLE_SCALETO_FUN(F16, F16,  vxc_short8,  vxc_half8,   half4,  short4, CONV)\n\
UPSAMPLE_SCALETO_FUN(F16, I16,  vxc_short8,  vxc_half8,   int4,   short4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(F16, I8,   vxc_short8,  vxc_half8,   int4,   char4,  CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(F16, U8,   vxc_short8,  vxc_half8,   int4,   uchar4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(I16, I16,  vxc_short8,  vxc_short8,  int4,   short4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(I16, F16,  vxc_short8,  vxc_short8,  half4,  short4, CONV)\n\
UPSAMPLE_SCALETO_FUN(I8,  I8,   vxc_char16,  vxc_char16,  int4,   char4,  CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(I8,  F16,  vxc_short8,  vxc_short8,  half4,  short4, CONV)\n\
UPSAMPLE_SCALETO_FUN(U8,  U8,   vxc_uchar16, vxc_uchar16, int4,   uchar4, CONV_RTE)\n\
UPSAMPLE_SCALETO_FUN(U8,  F16,  vxc_short8,  vxc_short8,  half4,  short4, CONV)\n\
\n\
"; /* end of upsamplescale_vx*/

static const char upsamplescale_k2_vx[] = "\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniUpScale2X_lo_2x8;\n\
_viv_uniform VXC_512Bits uniUpScale2X_hi_2x8;\n\
_viv_uniform int2 multAndoutZP;//[0:15] multiplier, [31:63] output zp\n\
\n\
#define UPSAMPLE_SCALETO8B_FUN(src_name, dst_name, read_type, src_type, dst_type) \\\n\
    __kernel void upsamplescale_##src_name##to##dst_name##_K2( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride, \\\n\
                 float            scale) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst_val; \\\n\
    VXC_ReadImage2DArray(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    coord.xy <<= 1; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.w, baseAddr); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst_val, src_val, multiplier, \\\n\
          VXC_MODIFIER(0, 7, 0, VXC_RM_ToNearestEven, 1), uniUpScale2X_lo_2x8); \\\n\
    VXC_DP2x8(dst_val, src_val, multiplier, \\\n\
          VXC_MODIFIER(8, 15, 0, VXC_RM_ToNearestEven, 1), uniUpScale2X_hi_2x8); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst_val, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.y ++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, dst_val, VXC_MODIFIER(0, 15, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
\n\
UPSAMPLE_SCALETO8B_FUN(F16, I8,  vxc_short8,  vxc_half8,   vxc_char16)\n\
UPSAMPLE_SCALETO8B_FUN(F16, U8,  vxc_short8,  vxc_half8,   vxc_uchar16)\n\
UPSAMPLE_SCALETO8B_FUN(I8,  I8,  vxc_char16,  vxc_char16,  vxc_char16)\n\
UPSAMPLE_SCALETO8B_FUN(U8,  U8,  vxc_uchar16, vxc_uchar16, vxc_uchar16)\n\
\n\
#define UPSAMPLE_SCALETO16B_FUN(src_name, dst_name, read_type, src_type, dst_type, write_type) \\\n\
    __kernel void upsamplescale_##src_name##to##dst_name##_K2( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output, \\\n\
                 int              stride, \\\n\
                 float            scale) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    read_type  read_val; \\\n\
    src_type   src_val; \\\n\
    dst_type   dst0_val; \\\n\
    dst_type   dst1_val; \\\n\
    write_type write_val; \\\n\
    VXC_ReadImage2DArray(read_val, input, coord, VXC_5BITOFFSET_XY(0, 0), \\\n\
                VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, src_val, read_val, 16); \\\n\
    coord.xy <<= 1; \\\n\
    int8 output_desc; \\\n\
    _viv_asm(COPY, output_desc, output, sizeof(output_desc)); \\\n\
    int baseAddr = (int)coord.z * output_desc.s4 + output_desc.s0; \\\n\
    _viv_asm(MOV, coord.w, baseAddr); \\\n\
    vxc_ushort8 multiplier; \\\n\
    _viv_asm(COPY, multiplier, multAndoutZP, 16); \\\n\
    VXC_DP2x8(dst0_val, src_val, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniUpScale2X_lo_2x8); \\\n\
    VXC_DP2x8(dst1_val, src_val, multiplier, VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 1), uniUpScale2X_hi_2x8); \\\n\
    _viv_asm(COPY, write_val, dst0_val, 16); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.y ++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, write_val, dst1_val, 16); \\\n\
    coord.xy = coord.xy + (int2)(8, -1); \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
    coord.y ++; \\\n\
    VXC_OP4_NoDest(img_store_3d, output, coord.xywz, write_val, VXC_MODIFIER(0, 7, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
UPSAMPLE_SCALETO16B_FUN(F16, F16,  vxc_short8,  vxc_half8,   vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(F16, I16,  vxc_short8,  vxc_half8,   vxc_short8, vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(I8,  F16,  vxc_char16,  vxc_char16,  vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(U8,  F16,  vxc_uchar16, vxc_uchar16, vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(I16, F16,  vxc_short8,  vxc_short8,  vxc_half8,  vxc_short8)\n\
UPSAMPLE_SCALETO16B_FUN(I16, I16,  vxc_short8,  vxc_short8,  vxc_short8, vxc_short8)\n\
"; /* end of upsamplescale_k2_vx*/

static const char vsi_nn_kernel_header_vx[] = "/*\n\
 ============================================================================\n\
 Name        : libNNExt.vx\n\
 Author      : VSI\n\
 Version     :\n\
 Copyright   : Your copyright notice\n\
 Description :\n\
 ============================================================================\n\
 */\n\
#include \"cl_viv_vx_ext.h\"\n\
\n\
typedef struct Image\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
} Image;\n\
\n\
inline uchar* get_image_ptr_from_coord(Image img, int2 coord)\n\
{\n\
    return img.ptr + coord.x * img.stride_x + coord.y * img.stride_y;\n\
}\n\
\n\
inline Image create_image_from_image2d(image2d_t input, int stride_x)\n\
{\n\
    int stride_y;\n\
#if (USE_40BITS_VA==0)\n\
    int8 desc;\n\
#else\n\
    int8 desc;\n\
    _viv_asm(GET_IMAGE_STRIDE, stride_y, input);\n\
#endif\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
    uint address = as_uint(desc.s0);\n\
\n\
#if (USE_40BITS_VA==0)\n\
    stride_y = desc.s1;\n\
#endif\n\
\n\
    Image img =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = stride_y\n\
    };\n\
\n\
    return img;\n\
}\n\
\n\
typedef struct Tensor\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
    int             stride_z;\n\
} Tensor;\n\
\n\
inline uchar* get_tensor_ptr_from_coord(Tensor t, int4 coord)\n\
{\n\
    return t.ptr + coord.x * t.stride_x + coord.y * t.stride_y + coord.z * t.stride_z;\n\
}\n\
\n\
inline Tensor create_tensor_from_image2d_array(image2d_array_t input, int stride_x)\n\
{\n\
    int8 desc;\n\
    int2 strides;\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
#if (USE_40BITS_VA==0)\n\
    strides.x = desc.s1;\n\
    strides.y = desc.s4;\n\
#else\n\
    _viv_asm(GET_IMAGE_STRIDE, strides, input);\n\
#endif\n\
    uint address = as_uint(desc.s0);\n\
\n\
    Tensor t =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = strides.x,\n\
        .stride_z                      = strides.y\n\
    };\n\
\n\
    return t;\n\
}\n\
\n\
#if (VX_VERSION==1)\n\
#define VXC_DP2x8_b_(dst, src0, src1, src2, info, uniform)\\\n\
do\\\n\
{\\\n\
    _viv_asm(COPY, dst, src0, 16); \\\n\
} while (0)\n\
\n\
#define VXC_VertMin3_Integer(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    typeof (dst) tmp; \\\n\
    tmp  = min(src0, src1);\\\n\
    dst  = min(src2, tmp);\\\n\
} while (0)\n\
\n\
#define VXC_VertMin3_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    vxc_short8 val0_ver1, val1_ver1, val2_ver1, minVal_ver1, maxVal_ver1;\\\n\
    _viv_asm(COPY, val0_ver1, src0, 16);\\\n\
    _viv_asm(COPY, val1_ver1, src1, 16);\\\n\
    _viv_asm(COPY, val2_ver1, src2, 16);\\\n\
    maxVal_ver1 = max(val0_ver1, val1_ver1);\\\n\
    minVal_ver1 = min(val0_ver1, val1_ver1);\\\n\
    minVal_ver1 = maxVal_ver1 < 0 ? maxVal_ver1 : minVal_ver1; \\\n\
    maxVal_ver1 = max(val2_ver1, minVal_ver1);\\\n\
    minVal_ver1 = min(val2_ver1, minVal_ver1);\\\n\
    minVal_ver1 = maxVal_ver1 < 0 ? maxVal_ver1 : minVal_ver1; \\\n\
    _viv_asm(COPY, dst, minVal_ver1, 16); \\\n\
} while (0)\n\
\n\
#define VXC_VertMax3_Integer(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    int endBin         = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    int sourceBin     = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    int mod1 = VXC_MODIFIER_CLAMP(startBin, endBin, sourceBin, 0);\\\n\
    typeof (dst) tmp;\\\n\
    tmp = max(src0, src1);\\\n\
    tmp = max(src2, tmp);\\\n\
    VXC_Clamp(dst, tmp, tmp, tmp, mod1);\\\n\
} while (0)\n\
\n\
#define VXC_VertMax3_Half(dst, src0, src1, src2, info)\\\n\
 do\\\n\
 {\\\n\
     vxc_short8 val0_ver1, val1_ver1, val2_ver1, minVal_ver1, maxVal_ver1;\\\n\
     _viv_asm(COPY, val0_ver1, src0, 16);\\\n\
     _viv_asm(COPY, val1_ver1, src1, 16);\\\n\
     _viv_asm(COPY, val2_ver1, src2, 16);\\\n\
     maxVal_ver1 = max(val0_ver1, val1_ver1);\\\n\
     maxVal_ver1 = max(val2_ver1, maxVal_ver1);\\\n\
     minVal_ver1 = min(val0_ver1, val1_ver1);\\\n\
     minVal_ver1 = min(val2_ver1, minVal_ver1);\\\n\
     maxVal_ver1 = maxVal_ver1 >= 0 ? maxVal_ver1 : minVal_ver1;\\\n\
     _viv_asm(COPY, dst, maxVal_ver1, 16); \\\n\
 } while (0)\n\
\n\
#define VXC_HorzMax3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    int endBin         = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    int sourceBin     = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    int clamp         = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Max, clamp);\\\n\
    VXC_OP4(filter, dst, src0, src0, src0, mod1);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMax3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    int endBin         = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    int sourceBin     = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    int clamp         = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Max, clamp);\\\n\
    int mod2 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Min, clamp);\\\n\
    vxc_short8 val0, minVal, maxVal;\\\n\
    _viv_asm(COPY, val0, src0, 16);\\\n\
    VXC_OP4(filter, maxVal, val0, val0, val0, mod1);\\\n\
    VXC_OP4(filter, minVal, val0, val0, val0, mod2);\\\n\
    maxVal = maxVal >= 0 ? maxVal : minVal;\\\n\
    _viv_asm(COPY, dst, maxVal, 16);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    int endBin         = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    int sourceBin     = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    int clamp         = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Min, clamp);\\\n\
    VXC_OP4(filter, dst, src0, src0, src0, mod1);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    int startBin     = (info & VXC_START_BIN_BITMASK) >> 12;\\\n\
    int endBin         = (info & VXC_END_BIN_BITMASK) >> 8;\\\n\
    int sourceBin     = (info & VXC_SOURCE_BIN_BITMASK) >> 4;\\\n\
    int clamp         = (info & VXC_CLAMP_BITMASK) >> 22;\\\n\
    int mod1 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Max, clamp);\\\n\
    int mod2 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Min, clamp);\\\n\
    int mod3 = VXC_MODIFIER_FILTER(startBin, endBin, sourceBin, VXC_FM_Median, clamp);\\\n\
    vxc_short8 val0, minVal, maxVal, midVal;\\\n\
    _viv_asm(COPY, val0, src0, 16);\\\n\
    VXC_OP4(filter, maxVal, val0, val0, val0, mod1);\\\n\
    VXC_OP4(filter, minVal, val0, val0, val0, mod2);\\\n\
    VXC_OP4(filter, midVal, val0, val0, val0, mod3);\\\n\
    minVal = midVal  < 0 ? midVal : minVal;\\\n\
    minVal = maxVal  < 0 ? maxVal : minVal;\\\n\
    _viv_asm(COPY, dst, minVal, 16);\\\n\
} while (0)\n\
\n\
#define VXC_Clamp_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    typeof (dst) tmp;\\\n\
    VXC_VertMax3_Half(tmp, src0, src0, src1, info);\\\n\
    VXC_VertMin3_Half(dst, tmp, tmp, src2, info);\\\n\
} while (0)\n\
\n\
#else\n\
#define VXC_DP2x8_b_(dst, src0, src1, src2, info, uniform)\\\n\
do\\\n\
{\\\n\
    VXC_DP2x8_b(dst, src0, src1, src2, info, uniform); \\\n\
} while (0)\n\
\n\
#define VXC_VertMin3_Integer(dst, src0, src1, src2, info)\\\n\
 do\\\n\
 {\\\n\
    VXC_VertMin3(dst, src0, src1, src2, info);\\\n\
 } while (0)\n\
\n\
#define VXC_VertMin3_Half(dst, src0, src1, src2, info)\\\n\
 do\\\n\
 {\\\n\
    VXC_VertMin3(dst, src0, src1, src2, info);\\\n\
 } while (0)\n\
\n\
#define VXC_VertMax3_Integer(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    VXC_VertMax3(dst, src0, src1, src2, info);\\\n\
} while (0)\n\
\n\
#define VXC_VertMax3_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    VXC_VertMax3(dst, src0, src1, src2, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMax3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMax3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMax3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMax3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Integer(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMin3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_HorzMin3_Half(dst, src0, info)\\\n\
do\\\n\
{\\\n\
    VXC_HorzMin3(dst, src0, info);\\\n\
} while (0)\n\
\n\
#define VXC_Clamp_Half(dst, src0, src1, src2, info)\\\n\
do\\\n\
{\\\n\
    VXC_Clamp(dst, src0, src1, src2, info);\\\n\
} while (0)\n\
#endif\n\
"; /* end of vsi_nn_kernel_header_vx*/

static const char warp_affine_vx[] = "#include \"cl_viv_vx_ext.h\"\n\
\n\
_viv_uniform VXC_512Bits uniConvertDatatoF32_0_4x4;\n\
_viv_uniform VXC_512Bits uniConvertDatatoF32_1_4x4;\n\
_viv_uniform VXC_512Bits uniExtract8Data_2x8;\n\
_viv_uniform float input_scale;\n\
_viv_uniform float input_tail;\n\
_viv_uniform float output_scale;\n\
_viv_uniform float output_zp;\n\
#define WARP_AFFINE_SH_IMPL(name0, name1, src_type, src_copy_type, convert_type, dst_type, dst_copy_type) \\\n\
__kernel void warp_affine_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_t       matrix, \\\n\
    __write_only image2d_array_t output \\\n\
    ) \\\n\
{ \\\n\
    int4   coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), get_global_id(2)); \\\n\
    int4   coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0) + 1, get_global_id(1)); \\\n\
 \\\n\
    float4 coord_f = convert_float4(coord_in); \\\n\
 \\\n\
    int2 m_coord = (int2)(0, 0); \\\n\
    vxc_ushort8 m0, m1; \\\n\
    VXC_ReadImage(m0, matrix, m_coord, VXC_5BITOFFSET_XY(0, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_ReadImage(m1, matrix, m_coord, VXC_5BITOFFSET_XY(8, 0), VXC_MODIFIER(0, 7, 0, VXC_RM_TowardZero, 0)); \\\n\
    float4 matrix0, matrix1; \\\n\
    _viv_asm(COPY, matrix0, m0, 16); \\\n\
    _viv_asm(COPY, matrix1, m1, 16); \\\n\
 \\\n\
    coord_f = coord_f.xxzz * matrix0.xyxy + coord_f.yyww * matrix0.zwzw + matrix1.xyxy; \\\n\
 \\\n\
    coord_in = convert_int4(coord_f < 0 ? coord_f - 2 : coord_f); \\\n\
 \\\n\
    int4 coord_in0 = (int4)(coord_in.xy, coord.zw); \\\n\
    int8 input_desc; \\\n\
    _viv_asm(COPY, input_desc, input, sizeof(input_desc)); \\\n\
    int baseAddr = (int)coord_in0.z * input_desc.s4 + input_desc.s0; \\\n\
    _viv_asm(MOV, coord_in0.z, baseAddr); \\\n\
 \\\n\
    src_type v0, v1; \\\n\
    src_copy_type top, bot; \\\n\
    VXC_OP4(img_load_3d, v0, input, coord_in0, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, v1, input, coord_in0, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(0, 1, 0, VXC_RM_TowardZero, 0)); \\\n\
    coord_in0.xy = coord_in.zw; \\\n\
    VXC_OP4(img_load_3d, v0, input, coord_in0, VXC_5BITOFFSET_XY(0, 0), \\\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    VXC_OP4(img_load_3d, v1, input, coord_in0, VXC_5BITOFFSET_XY(0, 1), \\\n\
            VXC_MODIFIER(2, 3, 0, VXC_RM_TowardZero, 0)); \\\n\
    _viv_asm(COPY, top, v0, 16); \\\n\
    _viv_asm(COPY, bot, v1, 16); \\\n\
 \\\n\
    float4 lerp       = coord_f - floor(coord_f); \\\n\
    float4 minus_lerp = 1.0f - lerp; \\\n\
    float4 coef0          = (float4)( minus_lerp.x * minus_lerp.y, lerp.x * minus_lerp.y, \\\n\
                                      minus_lerp.x * lerp.y,       lerp.x * lerp.y); \\\n\
 \\\n\
    float4 coef1          = (float4)( minus_lerp.z * minus_lerp.w, lerp.z * minus_lerp.w, \\\n\
                               minus_lerp.z * lerp.w,       lerp.z * lerp.w); \\\n\
 \\\n\
    float4 data0, data1, result = 0; \\\n\
    VXC_DP4x4(data0, top, bot, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDatatoF32_0_4x4); \\\n\
    VXC_DP4x4(data1, top, bot, VXC_MODIFIER(0, 3, 0, VXC_RM_TowardZero, 0), uniConvertDatatoF32_1_4x4); \\\n\
 \\\n\
    data0 = data0 * input_scale + input_tail; \\\n\
    data1 = data1 * input_scale + input_tail; \\\n\
    result.x = dot(data0, coef0); \\\n\
    result.y = dot(data1, coef1); \\\n\
    result.xy = result.xy * output_scale + output_zp; \\\n\
    convert_type dst0; \\\n\
    _viv_asm(CONV_RTE, dst0, result); \\\n\
    dst_type dst1; \\\n\
    VXC_DP2x8(dst1, dst0, dst0, VXC_MODIFIER(0, 1, 0, VXC_RM_ToNearestEven, 1), uniExtract8Data_2x8); \\\n\
    dst_copy_type dst; \\\n\
    _viv_asm(COPY, dst, dst1, 8); \\\n\
    VXC_WriteImage2DArray(output, coord, dst, VXC_MODIFIER(0, 1, 0,VXC_RM_TowardZero, 0)); \\\n\
}\n\
WARP_AFFINE_SH_IMPL(F16, F16, vxc_short8, vxc_half8,  half4, vxc_half8,  vxc_short8)\n\
WARP_AFFINE_SH_IMPL(F16, I8,  vxc_short8, vxc_half8,  int4,  vxc_char8,  vxc_char8)\n\
WARP_AFFINE_SH_IMPL(F16, U8,  vxc_short8, vxc_half8,  int4,  vxc_uchar8, vxc_uchar8)\n\
WARP_AFFINE_SH_IMPL(F16, I16, vxc_short8, vxc_half8,  int4,  vxc_short8, vxc_short8)\n\
WARP_AFFINE_SH_IMPL(I8,  I8,  vxc_char8,  vxc_char8,  int4,  vxc_char8,  vxc_char8)\n\
WARP_AFFINE_SH_IMPL(I8,  F16, vxc_char8,  vxc_char8,  half4, vxc_half8,  vxc_short8)\n\
WARP_AFFINE_SH_IMPL(U8,  U8,  vxc_uchar8, vxc_uchar8, int4,  vxc_uchar8, vxc_uchar8)\n\
WARP_AFFINE_SH_IMPL(U8,  F16, vxc_uchar8, vxc_uchar8, half4, vxc_half8,  vxc_short8)\n\
WARP_AFFINE_SH_IMPL(I16, I16, vxc_short8, vxc_short8, int4,  vxc_short8, vxc_short8)\n\
WARP_AFFINE_SH_IMPL(I16, F16, vxc_short8, vxc_short8, half4, vxc_half8,  vxc_short8)\n\
"; /* end of warp_affine_vx*/



static const char add_mean_std_norm_cl[] = "\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_F32_F32toF32(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
    float rsEps, float dimRatio,\n\
    float input0Scale, float input0Tail,\n\
    float input1Scale, float input1Tail,\n\
    float outputScale, float outputZP,\n\
    int width)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    float4 src0, src1, result;\n\
    float pSum = 0.0f, pSqr = 0.0f;\n\
    float sum  = 0.0f,  sqr = 0.0f;\n\
    float input_d = 0.0f;\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = read_imagef(input, coord);\n\
        src1 = read_imagef(input1, coord);\n\
        input_d = src0.x + src1.x;\n\
        pSum += input_d;\n\
        pSqr += input_d * input_d;\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
    float mean;\n\
    mean = sum * dimRatio;\n\
    float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = read_imagef(input, coord);\n\
        src1 = read_imagef(input1, coord);\n\
        input_d = src0.x + src1.x;\n\
        result.x = input_d * stddev_inv + rMeanStd;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_U8_U8toF32(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
    float rsEps, float dimRatio,\n\
    float input0Scale, float input0Tail,\n\
    float input1Scale, float input1Tail,\n\
    float outputScale, float outputZP,\n\
    int width)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    float4 src0, src1, result;\n\
    float pSum = 0.0f, pSqr = 0.0f;\n\
    float sum  = 0.0f,  sqr = 0.0f;\n\
    float input_d = 0.0f;\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        pSum += input_d;\n\
        pSqr += input_d * input_d;\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
    float mean;\n\
    mean = sum * dimRatio;\n\
    float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        result.x = input_d * stddev_inv + rMeanStd;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void add_mean_std_norm_U8_U8toU8(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
    float rsEps, float dimRatio,\n\
    float input0Scale, float input0Tail,\n\
    float input1Scale, float input1Tail,\n\
    float outputScale, float outputZP,\n\
    int width)\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    float4 src0, src1, result = 0.0f;\n\
    float pSum = 0.0f, pSqr = 0.0f;\n\
    float sum  = 0.0f,  sqr = 0.0f;\n\
    float input_d = 0.0f;\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        pSum += input_d;\n\
        pSqr += input_d * input_d;\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    lcl_sqr[lidx] = pSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    pLocalPtr = (float4 *)&lcl_sqr[0];\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sqr = dot(data0, one);\n\
    float mean;\n\
    mean = sum * dimRatio;\n\
    float vari, stddev_inv, rMeanStd;\n\
    vari = sqr*dimRatio - mean*mean;\n\
    stddev_inv = (vari==0 ? rsEps : rsqrt(vari));\n\
    rMeanStd = (-mean) * stddev_inv;\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        src0 = convert_float4(read_imageui(input, coord))  * input0Scale - input0Tail;\n\
        src1 = convert_float4(read_imageui(input1, coord)) * input1Scale - input1Tail;\n\
        input_d = src0.x + src1.x;\n\
        result.x = input_d * stddev_inv + rMeanStd;\n\
        uint4 dst = convert_uint4(result * outputScale + outputZP);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
"; /* end of add_mean_std_norm_cl*/

static const char argmax_axis0_cl[] = "__kernel void argmax_axis0_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.x : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
"; /* end of argmax_axis0_cl*/

static const char argmax_axis1_cl[] = "__kernel void argmax_axis1_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.y : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
"; /* end of argmax_axis1_cl*/

static const char argmax_axis2_cl[] = "__kernel void argmax_axis2_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.z : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.z : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x > minVal.x ? coord.z : minIdx;\n\
        minVal = val > minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmax_axis2_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
"; /* end of argmax_axis2_cl*/

static const char argmin_axis0_cl[] = "__kernel void argmin_axis0_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(0, get_global_id(0));\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.x : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minIdx);\n\
}\n\
\n\
"; /* end of argmin_axis0_cl*/

static const char argmin_axis1_cl[] = "__kernel void argmin_axis1_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.y : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minIdx);\n\
}\n\
\n\
"; /* end of argmin_axis1_cl*/

static const char argmin_axis2_cl[] = "__kernel void argmin_axis2_F32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.z : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_F32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_U8toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    uint4 minVal = read_imageui(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.z : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_U8toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                         int     axisSize\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    int minIdx = 0;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minIdx = val.x < minVal.x ? coord.z : minIdx;\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
__kernel void argmin_axis2_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                       int axisSize\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int minIdx = 0;\n\
\n\
    write_imagei(output, coord.xy, minIdx);\n\
}\n\
\n\
"; /* end of argmin_axis2_cl*/

static const char batchnorm_single_cl[] = "#define BN_U8_SAVE \\\n\
    uint4 dst = convert_uint4(src * output_scale + output_zp); \\\n\
    write_imageui(output, coord, dst);\n\
\n\
#define BN_I32_SAVE \\\n\
    int4 dst = convert_int4(src * output_scale + output_zp); \\\n\
    write_imagei(output, coord, dst);\n\
\n\
#define BN_F32_SAVE \\\n\
    write_imagef(output, coord, src);\n\
\n\
#define BATCH_NORM_F32_SH_IMPL(TYPE) \\\n\
__kernel void batch_norm_F32to##TYPE \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 float           output_scale, \\\n\
                 float           output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src, mean, var, gamma, beta; \\\n\
    READ_IMAGEF_2DARRAY(src, input, coord); \\\n\
    READ_IMAGEF_2DARRAY(mean, Mean, coord); \\\n\
    READ_IMAGEF_2DARRAY(var, Variance, coord); \\\n\
    READ_IMAGEF_2DARRAY(gamma, Gamma, coord); \\\n\
    READ_IMAGEF_2DARRAY(beta, Beta, coord); \\\n\
 \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_F32_SH_IMPL(F32)\n\
BATCH_NORM_F32_SH_IMPL(U8)\n\
BATCH_NORM_F32_SH_IMPL(I32)\n\
\n\
#define BATCH_NORM_F32_SH_IMPL_2D(TYPE) \\\n\
__kernel void batch_norm_F32to##TYPE##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t Mean, \\\n\
    __read_only  image2d_t Variance, \\\n\
    __read_only  image2d_t Gamma, \\\n\
    __read_only  image2d_t Beta, \\\n\
    __write_only image2d_t output, \\\n\
                 float     eps, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 float     output_scale, \\\n\
                 float     output_zp \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
    float4 mean = read_imagef(Mean, coord); \\\n\
    float4 var = read_imagef(Variance, coord); \\\n\
    float4 gamma = read_imagef(Gamma, coord); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_F32_SH_IMPL_2D(F32)\n\
BATCH_NORM_F32_SH_IMPL_2D(U8)\n\
BATCH_NORM_F32_SH_IMPL_2D(I32)\n\
\n\
#define BATCH_NORM_U8_SH_IMPL(TYPE) \\\n\
__kernel void batch_norm_U8to##TYPE \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 float           output_scale, \\\n\
                 float           output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 data; \\\n\
    float4 src, mean, var, gamma, beta; \\\n\
    READ_IMAGEUI_2DARRAY(data, input, coord); \\\n\
    READ_IMAGEF_2DARRAY(mean, Mean, coord); \\\n\
    READ_IMAGEF_2DARRAY(var, Variance, coord); \\\n\
    READ_IMAGEF_2DARRAY(gamma, Gamma, coord); \\\n\
    READ_IMAGEF_2DARRAY(beta, Beta, coord); \\\n\
 \\\n\
    src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_U8_SH_IMPL(U8)\n\
BATCH_NORM_U8_SH_IMPL(F32)\n\
\n\
#define BATCH_NORM_U8_SH_IMPL_2D(TYPE) \\\n\
__kernel void batch_norm_U8to##TYPE##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t Mean, \\\n\
    __read_only  image2d_t Variance, \\\n\
    __read_only  image2d_t Gamma, \\\n\
    __read_only  image2d_t Beta, \\\n\
    __write_only image2d_t output, \\\n\
                 float     eps, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 float     output_scale, \\\n\
                 float     output_zp \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4  data = read_imageui(input, coord); \\\n\
    float4 mean = read_imagef(Mean, coord); \\\n\
    float4 var = read_imagef(Variance, coord); \\\n\
    float4 gamma = read_imagef(Gamma, coord); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    float4 src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_U8_SH_IMPL_2D(U8)\n\
BATCH_NORM_U8_SH_IMPL_2D(F32)\n\
\n\
#define BATCH_NORM_I32_SH_IMPL(TYPE) \\\n\
__kernel void batch_norm_I32to##TYPE \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __read_only  image2d_array_t Mean, \\\n\
    __read_only  image2d_array_t Variance, \\\n\
    __read_only  image2d_array_t Gamma, \\\n\
    __read_only  image2d_array_t Beta, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           eps, \\\n\
                 float           input_scale, \\\n\
                 float           input_tail, \\\n\
                 float           output_scale, \\\n\
                 float           output_zp \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    int4 data; \\\n\
    float4 src, mean, var, gamma, beta; \\\n\
    READ_IMAGEI_2DARRAY(data, input, coord); \\\n\
    READ_IMAGEF_2DARRAY(mean, Mean, coord); \\\n\
    READ_IMAGEF_2DARRAY(var, Variance, coord); \\\n\
    READ_IMAGEF_2DARRAY(gamma, Gamma, coord); \\\n\
    READ_IMAGEF_2DARRAY(beta, Beta, coord); \\\n\
 \\\n\
    src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_I32_SH_IMPL(I32)\n\
BATCH_NORM_I32_SH_IMPL(F32)\n\
\n\
#define BATCH_NORM_I32_SH_IMPL_2D(TYPE) \\\n\
__kernel void batch_norm_I32to##TYPE##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __read_only  image2d_t Mean, \\\n\
    __read_only  image2d_t Variance, \\\n\
    __read_only  image2d_t Gamma, \\\n\
    __read_only  image2d_t Beta, \\\n\
    __write_only image2d_t output, \\\n\
                 float     eps, \\\n\
                 float     input_scale, \\\n\
                 float     input_tail, \\\n\
                 float     output_scale, \\\n\
                 float     output_zp \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    int4  data = read_imagei(input, coord); \\\n\
    float4 mean = read_imagef(Mean, coord); \\\n\
    float4 var = read_imagef(Variance, coord); \\\n\
    float4 gamma = read_imagef(Gamma, coord); \\\n\
    float4 beta = read_imagef(Beta, coord); \\\n\
 \\\n\
    float4 src = convert_float4(data) * input_scale - input_tail; \\\n\
    src.x = src.x - mean.x; \\\n\
    float inv = rsqrt(var.x + eps); \\\n\
    src.x = src.x * inv *gamma.x + beta.x; \\\n\
 \\\n\
    BN_##TYPE##_SAVE \\\n\
}\n\
BATCH_NORM_I32_SH_IMPL_2D(I32)\n\
BATCH_NORM_I32_SH_IMPL_2D(F32)"; /* end of batchnorm_single_cl*/

static const char cast_cl[] = "\n\
#define CAST_FUN(src_name, dst_name, src_type, dst_type, conv_fun, read_fun, write_fun) \\\n\
__kernel void cast_##src_name##to##dst_name( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    dst_type dst = 0; \\\n\
    dst = conv_fun(src); \\\n\
    write_fun(output, coord, dst); \\\n\
}\n\
\n\
CAST_FUN(F32, I32, float4, int4,   convert_int4_sat,  read_imagef,  write_imagei)\n\
CAST_FUN(F32, U32, float4, uint4,  convert_uint4_sat, read_imagef,  write_imageui)\n\
CAST_FUN(I32, I32, int4,   int4,   convert_int4_sat,  read_imagei,  write_imagei)\n\
CAST_FUN(I32, U32, int4,   uint4,  convert_uint4_sat, read_imagei,  write_imageui)\n\
CAST_FUN(U32, I32, uint4,  int4,   convert_int4_sat,  read_imageui, write_imagei)\n\
CAST_FUN(U32, U32, uint4,  uint4,  convert_uint4_sat, read_imageui, write_imageui)\n\
CAST_FUN(F32, F32, float4, float4, convert_float4,    read_imagef,  write_imagef)\n\
CAST_FUN(I32, F32, int4,   float4, convert_float4,    read_imagei,  write_imagef)\n\
CAST_FUN(U32, F32, uint4,  float4, convert_float4,    read_imageui, write_imagef)\n\
\n\
#define CAST_FUN_2D(src_name, dst_name, src_type, dst_type, conv_fun, read_fun, write_fun) \\\n\
__kernel void cast_##src_name##to##dst_name##_2D( \\\n\
    __read_only  image2d_t  input, \\\n\
    __write_only image2d_t  output) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    dst_type dst = 0; \\\n\
    dst = conv_fun(src); \\\n\
    write_fun(output, coord, dst); \\\n\
}\n\
\n\
CAST_FUN_2D(F32, I32, float4, int4,   convert_int4_sat,  read_imagef,  write_imagei)\n\
CAST_FUN_2D(F32, U32, float4, uint4,  convert_uint4_sat, read_imagef,  write_imageui)\n\
CAST_FUN_2D(I32, I32, int4,   int4,   convert_int4_sat,  read_imagei,  write_imagei)\n\
CAST_FUN_2D(I32, U32, int4,   uint4,  convert_uint4_sat, read_imagei,  write_imageui)\n\
CAST_FUN_2D(U32, I32, uint4,  int4,   convert_int4_sat,  read_imageui, write_imagei)\n\
CAST_FUN_2D(U32, U32, uint4,  uint4,  convert_uint4_sat, read_imageui, write_imageui)\n\
CAST_FUN_2D(F32, F32, float4, float4, convert_float4,    read_imagef,  write_imagef)\n\
CAST_FUN_2D(I32, F32, int4,   float4, convert_float4,    read_imagei,  write_imagef)\n\
CAST_FUN_2D(U32, F32, uint4,  float4, convert_float4,    read_imageui, write_imagef)\n\
\n\
#define CAST_TO_BOOL_FUN(src_name, src_type, read_fun) \\\n\
__kernel void cast_##src_name##toBOOL8( \\\n\
    __read_only  image2d_array_t  input, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    int4 dst = 0; \\\n\
    dst.x = (int)(src.x != 0); \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN(F32, float4, read_imagef)\n\
CAST_TO_BOOL_FUN(I32, int4,   read_imagei)\n\
CAST_TO_BOOL_FUN(U32, uint4,  read_imageui)\n\
\n\
\n\
#define CAST_TO_BOOL_FUN_2D(src_name, src_type, read_fun) \\\n\
__kernel void cast_##src_name##toBOOL8_2D( \\\n\
    __read_only  image2d_t  input, \\\n\
    __write_only image2d_t  output) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
    src_type src = read_fun(input, coord); \\\n\
    int4 dst = 0; \\\n\
    dst.x = (int)(src.x != 0); \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
CAST_TO_BOOL_FUN_2D(F32, float4, read_imagef)\n\
CAST_TO_BOOL_FUN_2D(I32, int4,   read_imagei)\n\
CAST_TO_BOOL_FUN_2D(U32, uint4,  read_imageui)\n\
\n\
"; /* end of cast_cl*/

static const char clip_F32_cl[] = "__kernel void clip_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = src > minData ? src : minData;\n\
    dst = dst < maxData ? dst : maxData;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  minData,\n\
                     float  maxData)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = src > minData ? src : minData;\n\
    dst = dst < maxData ? dst : maxData;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = src > minData ? src : minData;\n\
    result = result < maxData ? result : maxData;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_F32toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  minData,\n\
                     float  maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = src > minData ? src : minData;\n\
    result = result < maxData ? result : maxData;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
"; /* end of clip_F32_cl*/

static const char clip_U8_cl[] = "__kernel void clip_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src > minData ? src : minData;\n\
    result = result < maxData ? result : maxData;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_U8toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  minData,\n\
                     float  maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src > minData ? src : minData;\n\
    result = result < maxData ? result : maxData;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_U8toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  minData,\n\
                           float  maxData,\n\
                           float inputScale,\n\
                           float inputTail,\n\
                           float outputScale,\n\
                           float outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src > minData ? src : minData;\n\
    dst = dst < maxData ? dst : maxData;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void clip_U8toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  minData,\n\
                     float  maxData,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src > minData ? src : minData;\n\
    dst = dst < maxData ? dst : maxData;\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of clip_U8_cl*/

static const char detect_post_box_cl[] = "float exp_(float x, float logE)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
__kernel void detect_post_box_F32_F32toF32(\n\
     __read_only image2d_array_t   input0,\n\
           __read_only image2d_t   input1,\n\
    __write_only image2d_array_t   output,\n\
                           float   inv_scale_y,\n\
                           float   inv_scale_x,\n\
                           float   inv_scale_h,\n\
                           float   inv_scale_w,\n\
                           float   logE)\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    float4 dst;\n\
    float4 tmp0, tmp1;\n\
    src0.x = read_imagef(input0, coord).x;\n\
    src1.x = read_imagef(input1, coord.xy).x;\n\
    coord.x++;\n\
    src0.y = read_imagef(input0, coord).x;\n\
    src1.y = read_imagef(input1, coord.xy).x;\n\
    coord.x++;\n\
    src0.z = read_imagef(input0, coord).x;\n\
    src1.z = read_imagef(input1, coord.xy).x;\n\
    coord.x++;\n\
    src0.w = read_imagef(input0, coord).x;\n\
    src1.w = read_imagef(input1, coord.xy).x;\n\
\n\
    tmp0.x  = src1.x + src1.z * src0.x * inv_scale_y;\n\
    tmp0.y  = src1.y + src1.w * src0.y * inv_scale_x;\n\
    tmp1.x = src1.z * exp_(src0.z * inv_scale_h, logE) * 0.5f;\n\
    tmp1.y = src1.w * exp_(src0.w * inv_scale_w, logE) * 0.5f;\n\
    dst.xy = tmp0.xy - tmp1.xy;\n\
    dst.zw = tmp0.xy + tmp1.xy;\n\
    coord.x = 0;\n\
    write_imagef(output, coord, dst.xxxx);\n\
    coord.x++;\n\
    write_imagef(output, coord, dst.yyyy);\n\
    coord.x++;\n\
    write_imagef(output, coord, dst.zzzz);\n\
    coord.x++;\n\
    write_imagef(output, coord, dst.wwww);\n\
}\n\
\n\
\n\
__kernel void detect_post_box_U8_U8toF32(\n\
     __read_only image2d_array_t   input0,\n\
           __read_only image2d_t   input1,\n\
    __write_only image2d_array_t   output,\n\
                           float   inv_scale_y,\n\
                           float   inv_scale_x,\n\
                           float   inv_scale_h,\n\
                           float   inv_scale_w,\n\
                           float   logE,\n\
                           float   input0Tail,\n\
                           float   input1Tail,\n\
                           float   input0Scale,\n\
                           float   input1Scale)\n\
{\n\
    int4 coord =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    uint4  in0, in1;\n\
    float4 src0;\n\
    float4 src1;\n\
    float4 dst;\n\
    float4 tmp0, tmp1;\n\
    in0.x = read_imageui(input0, coord).x;\n\
    in1.x = read_imageui(input1, coord.xy).x;\n\
    coord.x++;\n\
    in0.y = read_imageui(input0, coord).x;\n\
    in1.y = read_imageui(input1, coord.xy).x;\n\
    coord.x++;\n\
    in0.z = read_imageui(input0, coord).x;\n\
    in1.z = read_imageui(input1, coord.xy).x;\n\
    coord.x++;\n\
    in0.w = read_imageui(input0, coord).x;\n\
    in1.w = read_imageui(input1, coord.xy).x;\n\
\n\
    src0 = convert_float4(in0) * input0Scale + input0Tail;\n\
    src1 = convert_float4(in1) * input1Scale + input1Tail;\n\
\n\
    tmp0.x  = src1.x + src1.z * src0.x * inv_scale_y;\n\
    tmp0.y  = src1.y + src1.w * src0.y * inv_scale_x;\n\
    tmp1.x = src1.z * exp_(src0.z * inv_scale_h, logE) * 0.5f;\n\
    tmp1.y = src1.w * exp_(src0.w * inv_scale_w, logE) * 0.5f;\n\
    dst.xy = tmp0.xy - tmp1.xy;\n\
    dst.zw = tmp0.xy + tmp1.xy;\n\
    coord.x = 0;\n\
    write_imagef(output, coord, dst.xxxx);\n\
    coord.x++;\n\
    write_imagef(output, coord, dst.yyyy);\n\
    coord.x++;\n\
    write_imagef(output, coord, dst.zzzz);\n\
    coord.x++;\n\
    write_imagef(output, coord, dst.wwww);\n\
}"; /* end of detect_post_box_cl*/

static const char eltwise_ops_helper_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
#pragma OPENCL EXTENSION cl_viv_vx_extension : enable\n\
\n\
typedef struct Image\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
} Image;\n\
\n\
inline uchar* get_image_ptr_from_coord(Image img, int2 coord)\n\
{\n\
    return img.ptr + coord.x * img.stride_x + coord.y * img.stride_y;\n\
}\n\
\n\
inline Image create_image_from_image2d(image2d_t input, int stride_x)\n\
{\n\
    int stride_y;\n\
#if (USE_40BITS_VA==0)\n\
    int8 desc;\n\
#else\n\
    int8 desc;\n\
    _viv_asm(GET_IMAGE_STRIDE, stride_y, input);\n\
#endif\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
    uint address = as_uint(desc.s0);\n\
\n\
#if (USE_40BITS_VA==0)\n\
    stride_y = desc.s1;\n\
#endif\n\
\n\
    Image img =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = stride_y\n\
    };\n\
\n\
    return img;\n\
}\n\
\n\
typedef struct Tensor\n\
{\n\
    __global uchar *ptr;\n\
    int             stride_x;\n\
    int             stride_y;\n\
    int             stride_z;\n\
} Tensor;\n\
\n\
inline uchar* get_tensor_ptr_from_coord(Tensor t, int4 coord)\n\
{\n\
    return t.ptr + coord.x * t.stride_x + coord.y * t.stride_y + coord.z * t.stride_z;\n\
}\n\
\n\
inline Tensor create_tensor_from_image2d_array(image2d_array_t input, int stride_x)\n\
{\n\
    int8 desc;\n\
    int2 strides;\n\
    _viv_asm(COPY, desc, input, sizeof(desc));\n\
\n\
#if (USE_40BITS_VA==0)\n\
    strides.x = desc.s1;\n\
    strides.y = desc.s4;\n\
#else\n\
    _viv_asm(GET_IMAGE_STRIDE, strides, input);\n\
#endif\n\
    uint address = as_uint(desc.s0);\n\
\n\
    Tensor t =\n\
    {\n\
        .ptr                           = (uchar*)(uintptr_t)address,\n\
        .stride_x                      = stride_x,\n\
        .stride_y                      = strides.x,\n\
        .stride_z                      = strides.y\n\
    };\n\
\n\
    return t;\n\
}\n\
\n\
#define READ_IMAGEF_2DARRAY(dest, tensor, coord) \\\n\
    do { \\\n\
        int depth = get_image_array_size(tensor); \\\n\
        int4 coord_in = coord; \\\n\
        _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
        dest = read_imagef(tensor, coord_in); \\\n\
       } while(0)\n\
\n\
#define READ_IMAGEI_2DARRAY(dest, tensor, coord) \\\n\
    do { \\\n\
        int depth = get_image_array_size(tensor); \\\n\
        int4 coord_in = coord; \\\n\
        _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
        dest = read_imagei(tensor, coord_in); \\\n\
       } while(0)\n\
\n\
#define READ_IMAGEUI_2DARRAY(dest, tensor, coord) \\\n\
    do { \\\n\
        int depth = get_image_array_size(tensor); \\\n\
        int4 coord_in = coord; \\\n\
        _viv_asm(CLAMP0MAX, coord_in.z, coord_in.z, depth - 1); \\\n\
        dest = read_imageui(tensor, coord_in); \\\n\
       } while(0)\n\
"; /* end of eltwise_ops_helper_cl*/

static const char eltwise_unary_cl[] = "float eltwise_unary_sin(float x, float alpha, float beta)\n\
{\n\
    return native_sin(x);\n\
}\n\
\n\
#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
float eltwise_unary_exp(float x, float alpha, float beta)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
\n\
#define rlogE    (0.693147182f)\n\
float eltwise_unary_log(float x, float alpha, float beta)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float eltwise_unary_elu(float val, float alpha, float beta)\n\
{\n\
    float x = val * logE;\n\
    x = exp2(x) * alpha - alpha;\n\
\n\
    return val < 0 ? x : val;\n\
}\n\
\n\
float eltwise_unary_neg(float x, float alpha, float beta)\n\
{\n\
    return x * -1;\n\
}\n\
\n\
float eltwise_unary_hard_sigmoid(float x, float alpha, float beta)\n\
{\n\
    x = alpha * x + beta;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
float _softrelu(float x, float alpha)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    x += 1;\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
float _tanh(float x, float alpha)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return (2 * x - 1);\n\
}\n\
\n\
float eltwise_unary_mish(float x, float alpha, float beta)\n\
{\n\
    float y = _softrelu(x, alpha);\n\
    x = x * _tanh(y, alpha);\n\
    return x;\n\
}\n\
\n\
float eltwise_unary_round(float x, float alpha, float beta)\n\
{\n\
    return convert_float(convert_int_rte(x));\n\
}\n\
\n\
#define MUL2_RSQRTPI    (1.1283791670955126f)\n\
float erf_eval(float x)\n\
{\n\
    float res = 0;\n\
    float tmp = x;\n\
    float factorial = 1;\n\
    float x_pow = x;\n\
    float one = 1.0f;\n\
    float n = 1;\n\
\n\
    if (x <= -3)\n\
        return -1;\n\
    else if (x >= 3)\n\
        return 1;\n\
\n\
    while (fabs(tmp) > 1e-5)\n\
    {\n\
        res += tmp;\n\
\n\
        factorial *= n;\n\
        one *= -1;\n\
        x_pow *= x * x;\n\
        tmp = one / factorial * x_pow / ( 2 * n + 1);\n\
\n\
        n += 1.0f;\n\
    }\n\
    return res * MUL2_RSQRTPI;\n\
}\n\
#define RSQRT2      (0.70710678118654752440084436210485f)\n\
float eltwise_unary_gelu(float x, float alpha, float beta)\n\
{\n\
    x = 0.5f * x * (1 + erf_eval(x * RSQRT2));\n\
\n\
    return x;\n\
}\n\
\n\
#define SQRT_2_RCP_PI  0.7978845834732056f\n\
float eltwise_unary_hard_gelu(float x, float alpha, float beta)\n\
{\n\
    float cdf = 0.5f + 0.5f * _tanh(SQRT_2_RCP_PI *\n\
                        (x + 0.044715f * x * x * x), 0);\n\
    return x * cdf;\n\
}\n\
\n\
#define ELTWISE_UNARY_F32(func_name) \\\n\
__kernel void func_name##_F32toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP, \\\n\
                 float           alpha, \\\n\
                 float           beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x, alpha, beta); \\\n\
 \\\n\
    write_imagef(output, coord, dst.xxxx); \\\n\
}\n\
ELTWISE_UNARY_F32(sin)\n\
ELTWISE_UNARY_F32(exp)\n\
ELTWISE_UNARY_F32(log)\n\
ELTWISE_UNARY_F32(elu)\n\
ELTWISE_UNARY_F32(neg)\n\
ELTWISE_UNARY_F32(mish)\n\
ELTWISE_UNARY_F32(hard_sigmoid)\n\
ELTWISE_UNARY_F32(round)\n\
ELTWISE_UNARY_F32(gelu)\n\
ELTWISE_UNARY_F32(hard_gelu)\n\
\n\
#define ELTWISE_UNARY_F32_2D(func_name) \\\n\
__kernel void func_name##_F32toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP, \\\n\
                 float     alpha, \\\n\
                 float           beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x, alpha, beta); \\\n\
 \\\n\
    write_imagef(output, coord, dst.xxxx); \\\n\
}\n\
ELTWISE_UNARY_F32_2D(sin)\n\
ELTWISE_UNARY_F32_2D(exp)\n\
ELTWISE_UNARY_F32_2D(log)\n\
ELTWISE_UNARY_F32_2D(elu)\n\
ELTWISE_UNARY_F32_2D(neg)\n\
ELTWISE_UNARY_F32_2D(mish)\n\
ELTWISE_UNARY_F32_2D(hard_sigmoid)\n\
ELTWISE_UNARY_F32_2D(round)\n\
ELTWISE_UNARY_F32_2D(gelu)\n\
ELTWISE_UNARY_F32_2D(hard_gelu)\n\
\n\
#define ELTWISE_UNARY_U8(func_name) \\\n\
__kernel void func_name##_U8toU8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP, \\\n\
                 float           alpha, \\\n\
                 float           beta \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x, alpha, beta); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8(sin)\n\
ELTWISE_UNARY_U8(exp)\n\
ELTWISE_UNARY_U8(log)\n\
ELTWISE_UNARY_U8(elu)\n\
ELTWISE_UNARY_U8(neg)\n\
ELTWISE_UNARY_U8(mish)\n\
ELTWISE_UNARY_U8(hard_sigmoid)\n\
ELTWISE_UNARY_U8(round)\n\
ELTWISE_UNARY_U8(gelu)\n\
ELTWISE_UNARY_U8(hard_gelu)\n\
\n\
#define ELTWISE_UNARY_U8_2D(func_name) \\\n\
__kernel void func_name##_U8toU8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP, \\\n\
                 float     alpha, \\\n\
                 float     beta \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x, alpha, beta); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8_2D(sin)\n\
ELTWISE_UNARY_U8_2D(exp)\n\
ELTWISE_UNARY_U8_2D(log)\n\
ELTWISE_UNARY_U8_2D(elu)\n\
ELTWISE_UNARY_U8_2D(neg)\n\
ELTWISE_UNARY_U8_2D(mish)\n\
ELTWISE_UNARY_U8_2D(hard_sigmoid)\n\
ELTWISE_UNARY_U8_2D(round)\n\
ELTWISE_UNARY_U8_2D(gelu)\n\
ELTWISE_UNARY_U8_2D(hard_gelu)\n\
\n\
__kernel void neg_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP,\n\
                 float           alpha,\n\
                 float           beta\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src = read_imagei(input, coord);\n\
\n\
    int4 dst = -src;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void neg_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail,\n\
                 float     outputScale,\n\
                 float     outputZP,\n\
                 float     alpha,\n\
                 float     beta\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src = read_imagei(input, coord);\n\
\n\
    int4 dst = -src;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of eltwise_unary_cl*/

static const char erf_cl[] = "#define MUL2_RSQRTPI    (1.1283791670955126f)\n\
float eltwise_unary_erf(float _x)\n\
{\n\
    float x = clamp(_x, -2, 2);\n\
    float res = 0;\n\
    float tmp = x;\n\
    float factorial = 1;\n\
    float x_pow = x;\n\
    float one = 1.0f;\n\
    float n = 1;\n\
\n\
    while (fabs(tmp) > 1e-5)\n\
    {\n\
        res += tmp;\n\
\n\
        factorial *= n;\n\
        one *= -1;\n\
        x_pow *= x * x;\n\
        tmp = one / factorial * x_pow / ( 2 * n + 1);\n\
\n\
        n += 1.0f;\n\
    }\n\
    return res * MUL2_RSQRTPI;\n\
}\n\
\n\
#define ELTWISE_UNARY_F32(func_name) \\\n\
__kernel void func_name##_F32toF32 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x); \\\n\
 \\\n\
    write_imagef(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_F32(erf)\n\
\n\
#define ELTWISE_UNARY_F32_2D(func_name) \\\n\
__kernel void func_name##_F32toF32_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src = read_imagef(input, coord); \\\n\
 \\\n\
    float4 dst = 0; \\\n\
    dst.x = eltwise_unary_##func_name(src.x); \\\n\
 \\\n\
    write_imagef(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_F32_2D(erf)\n\
\n\
#define ELTWISE_UNARY_U8(func_name) \\\n\
__kernel void func_name##_U8toU8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           inputScale, \\\n\
                 float           inputTail, \\\n\
                 float           outputScale, \\\n\
                 float           outputZP \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8(erf)\n\
\n\
#define ELTWISE_UNARY_U8_2D(func_name) \\\n\
__kernel void func_name##_U8toU8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                 float     inputScale, \\\n\
                 float     inputTail, \\\n\
                 float     outputScale, \\\n\
                 float     outputZP \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 src = read_imageui(input, coord); \\\n\
    float4 data = convert_float4(src) * inputScale - inputTail; \\\n\
 \\\n\
    data.x = eltwise_unary_##func_name(data.x); \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
 \\\n\
    write_imageui(output, coord, dst); \\\n\
}\n\
ELTWISE_UNARY_U8_2D(erf)\n\
"; /* end of erf_cl*/

static const char floordiv_cl[] = "__kernel void floordiv_F32F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
    float4 dst  = floor(src0 / src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_F32F32toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __read_only  image2d_t  input1,\n\
    __write_only image2d_t  output)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src0 = read_imagef(input, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
    float4 dst  = floor(src0 / src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toI32(\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    int4 dst  = convert_int4(floor(convert_float4(src0) / convert_float4(src1)));\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toI32_2D(\n\
    __read_only  image2d_t  input,\n\
    __read_only  image2d_t  input1,\n\
    __write_only image2d_t  output)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src0 = read_imagei(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    int4 dst  = convert_int4(floor(convert_float4(src0) / convert_float4(src1)));\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_I32I32toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __read_only  image2d_t  input1,\n\
    __write_only image2d_t  output,\n\
                 float      input0Scale,\n\
                 float      input0Tail,\n\
                 float      input1Scale,\n\
                 float      input1Tail,\n\
                 float      outputScale,\n\
                 float      outputTail )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src0 = read_imagei(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    float4 in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    float4 out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    uint4 src0, src1;\n\
    float4 in0, in1, out;\n\
    READ_IMAGEUI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst  = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8U8toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __read_only  image2d_t  input1,\n\
    __write_only image2d_t  output,\n\
                 float      input0Scale,\n\
                 float      input0Tail,\n\
                 float      input1Scale,\n\
                 float      input1Tail,\n\
                 float      outputScale,\n\
                 float      outputTail )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    uint4 src0 = read_imageui(input, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
    float4 in0, in1, out;\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1) * input1Scale + input1Tail;\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst  = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8I32toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail,\n\
                 float            outputScale,\n\
                 float            outputTail )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    uint4 src0;\n\
    int4 src1;\n\
    float4 in0, in1, out;\n\
    READ_IMAGEUI_2DARRAY(src0, input, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1);\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void floordiv_U8I32toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __read_only  image2d_t  input1,\n\
    __write_only image2d_t  output,\n\
                 float      input0Scale,\n\
                 float      input0Tail,\n\
                 float      input1Scale,\n\
                 float      input1Tail,\n\
                 float      outputScale,\n\
                 float      outputTail )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    uint4 src0 = read_imageui(input, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
    float4 in0, in1, out;\n\
    in0 = convert_float4(src0) * input0Scale + input0Tail;\n\
    in1 = convert_float4(src1);\n\
    out = floor(in0 / in1) * outputScale + outputTail;\n\
    uint4 dst = convert_uint4(out);\n\
    write_imageui(output, coord, dst);\n\
}\n\
"; /* end of floordiv_cl*/

static const char gather_cl[] = "__kernel void gather_U8toU8(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    uint4 data = read_imageui(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void gather_F16toF16(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imagef(output, coord, data);\n\
}\n\
\n\
__kernel void gather_I32toI32(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    int4 data = read_imagei(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imagei(output, coord, data);\n\
}\n\
\n\
__kernel void gather_F32toF32(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int block_num,\n\
    int axis_num,\n\
    int indices_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord_in = (int4)(gidy, 0, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord_in.xy);\n\
    coord_in.w = gidz * axis_num + indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord_in.zw);\n\
\n\
    int2 coord = (int2)(gidx, gidz * indices_num + gidy);\n\
    write_imagef(output, coord, data);\n\
}\n\
"; /* end of gather_cl*/

static const char gather_nd_cl[] = "__kernel void gather_nd_U8toU8_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    uint4 data = read_imageui(input0, coord.zw);\n\
    write_imageui(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord.zw);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_I32toI32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    int4 data = read_imagei(input0, coord.zw);\n\
    write_imagei(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F32toF32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 0);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    coord.w = indice.x;\n\
\n\
    float4 data = read_imagef(input0, coord.zw);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
//2D\n\
__kernel void gather_nd_U8toU8_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    uint4 data = read_imageui(input0, indice.xy);\n\
    write_imageui(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    float4 data = read_imagef(input0, indice.xy);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_I32toI32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    int4 data = read_imagei(input0, indice.xy);\n\
    write_imagei(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F32toF32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, gidx, 1);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.wy);\n\
    indice.x = indice.x * block_size + gidx;\n\
    indice.y = indice1.x;\n\
\n\
    float4 data = read_imagef(input0, indice.xy);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
"; /* end of gather_nd_cl*/

static const char gather_nd_3d_cl[] = "__kernel void gather_nd_U8toU8_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    uint4 data = read_imageui(input0, indice);\n\
    write_imageui(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F16toF16_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    float4 data = read_imagef(input0, indice);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_I32toI32_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    int4 data = read_imagei(input0, indice);\n\
    write_imagei(output, coord.zy, data);\n\
}\n\
\n\
__kernel void gather_nd_F32toF32_3D(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int block_size,\n\
    int coord_dim\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
    int gidz = get_global_id(2);  // block_num\n\
\n\
    int4 coord = (int4)(0, gidy, 1, 2);\n\
    int4 indice = read_imagei(input1, coord.xy);\n\
    int4 indice1 = read_imagei(input1, coord.zy);\n\
    int4 indice2 = read_imagei(input1, coord.wy);\n\
    indice = (int4)(indice.x * block_size + gidx, indice1.x, indice2.x, 0);\n\
    coord.z = gidx;\n\
\n\
    float4 data = read_imagef(input0, indice);\n\
    write_imagef(output, coord.zy, data);\n\
}\n\
"; /* end of gather_nd_3d_cl*/

static const char group_normalization_f32_cl[] = "__kernel void group_norm_sumsqr_F32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum += data.x;\n\
            sqr += data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_sumsqr_F32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        data = read_imagef(input, coord);\n\
        sum = data.x;\n\
        sqr = data.x * data.x;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_meanvari(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    float group_ratio,\n\
    int group_stride\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
\n\
    float2 sum_sqr = (float2)(0);\n\
    float4 mean_vari = (float4)(0);\n\
\n\
    __local float2 lcl_data[16];\n\
    __local float2 lcl_sum[4];\n\
\n\
    for(; coord.x < group_stride;)\n\
    {\n\
        mean_vari.x += read_imagef(input, coord).x;\n\
        coord.x++;\n\
        mean_vari.y += read_imagef(input, coord).x;\n\
        coord.x+=63;\n\
    }\n\
    lcl_data[lidx] = mean_vari.xy;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx < 4)\n\
    {\n\
        float2 tmpSum = (float2)(0);\n\
        for(int i = lidx; i < 16; i+=4)\n\
        {\n\
            tmpSum += lcl_data[i];\n\
        }\n\
        lcl_sum[lidx] = tmpSum;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx == 0)\n\
    {\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum_sqr += lcl_sum[i];\n\
        }\n\
        mean_vari.xy = sum_sqr * group_ratio;\n\
        mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
        mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
        coord.x = 0;\n\
        write_imagef(output, coord, mean_vari);\n\
        coord.x++;\n\
        float4 data;\n\
        data.x = mean_vari.y;\n\
        write_imagef(output, coord, data);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_F32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = read_imagef(input, coord);\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 dst;\n\
\n\
    dst.x = data.x * scale_vari + bias_val;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_F32toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = read_imagef(input, coord);\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = beta.s0 - scale_vari * mean_vari.s0;\n\
\n\
    float4 dst;\n\
\n\
    dst.x = data.x * scale_vari + bias_val;\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of group_normalization_f32_cl*/

static const char group_normalization_i32_cl[] = "__kernel void group_norm_sumsqr_I32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = convert_float4(read_imagei(input, coord));\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            sqr += (data.x * data.x * e2InScale);\n\
        }\n\
        sum = tmpSum * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_sumsqr_I32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        data = convert_float4(read_imagei(input, coord));\n\
        sum = data.x * input_scale;\n\
        sqr = sum * sum;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_I32toI32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale;\n\
\n\
    int4 dst;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_int4_rte(norm);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_I32toI32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * output_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale;\n\
\n\
    int4 dst;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_int4_rte(norm);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
\n\
__kernel void group_norm_I32toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imagei(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_scale * scale_vari;\n\
    bias_val = beta.s0 - scale_vari * mean_vari.s0;\n\
\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
"; /* end of group_normalization_i32_cl*/

static const char group_normalization_u8_cl[] = "__kernel void group_norm_sumsqr_U8(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
    float tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = convert_float4(read_imageui(input, coord));\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            tmpSqr += data.x * data.x;\n\
        }\n\
        sqr = (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum = (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_sumsqr_U8_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int2 coord = (int2)(gidx, gidz);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        data = convert_float4(read_imageui(input, coord));\n\
        sum = (data.x - input_zp) * input_scale;\n\
        sqr = sum * sum;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void group_norm_U8toU8(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 dst;\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_uint4_rte(norm);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_U8toU8_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 dst;\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    dst = convert_uint4_rte(norm);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void group_norm_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidy = get_global_id(1);\n\
    int gidz = get_global_id(2);\n\
    int4 coord = (int4)(get_global_id(0), gidy, gidz, 0);\n\
    int4 coord_para = (int4)((convert_int(get_global_id(0) * rSpaceOrg) + gidy * pStride), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
\n\
__kernel void group_norm_U8toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int is2d,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float rSpaceOrg,\n\
    int width,\n\
    int height,\n\
    int pStride\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int2 coord = (int2)(get_global_id(0), gidz);\n\
    int4 coord_para = (int4)(convert_int(get_global_id(0) * rSpaceOrg), gidz, 0, 1);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.xy);\n\
    float4 beta  = read_imagef(bias, coord_para.xy);\n\
    float4 mean_vari = read_imagef(meanVari, coord_para.zy);\n\
    mean_vari.y = read_imagef(meanVari, coord_para.wy).x;\n\
    float4 data = convert_float4(read_imageui(input, coord));\n\
\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    data.x -= input_zp;\n\
    float4 norm;\n\
    norm.x = data.x * alpha + bias_val;\n\
    write_imagef(output, coord, norm);\n\
}\n\
"; /* end of group_normalization_u8_cl*/

static const char grucell_activation_cl[] = "__kernel void grucell_activation(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
\n\
}\n\
"; /* end of grucell_activation_cl*/

static const char grucell_activation_sma_cl[] = "__kernel void grucell_activation_sma(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
\n\
}\n\
"; /* end of grucell_activation_sma_cl*/

static const char grucell_activation_z_h_cl[] = "#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float sigmoid(float x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float hard_sigmoid(float x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float tanh_func(float x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define GRUCELL_ACTIVATION_U8_F32_U8(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_U8_F32toU8_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 h_tm = convert_float4(read_imageui(hstate_in, coord_in.xy)); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 h = h0 + h1; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    dst = dst * output_scale + output_zp; \\\n\
    uint4 result = convert_uint4_sat_rte(dst); \\\n\
    write_imageui(output, coord_in.xy, result); \\\n\
    write_imageui(hstate_out, coord_in.xy, result); \\\n\
}\n\
GRUCELL_ACTIVATION_U8_F32_U8(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_ACTIVATION_F32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_F32_F32toF32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
    float4 h_tm = read_imagef(hstate_in, coord_in.xy); \\\n\
 \\\n\
    float4 h = h0 + h1; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    write_imagef(output, coord_in.xy, dst); \\\n\
    write_imagef(hstate_out, coord_in.xy, dst); \\\n\
}\n\
\n\
GRUCELL_ACTIVATION_F32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_ACTIVATION_I32_F32_I32(act_name, act_func) \\\n\
__kernel void grucell_activation_z_h_I32_F32toI32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 h_tm = convert_float4(read_imagei(hstate_in, coord_in.xy)); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 h = h0 + h1; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    dst = dst * output_scale + output_zp; \\\n\
    int4 result = convert_int4_sat_rte(dst); \\\n\
    write_imagei(output, coord_in.xy, result); \\\n\
    write_imagei(hstate_out, coord_in.xy, result); \\\n\
}\n\
GRUCELL_ACTIVATION_I32_F32_I32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)"; /* end of grucell_activation_z_h_cl*/

static const char grucell_h_times_activation_r_cl[] = "#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float sigmoid(float x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float hard_sigmoid(float x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
\n\
#define GRUCELL_H_TIMES_R_U8_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_U8_F32toF32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    float input_scale, float input_tail) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imageui(hstate_in, coord_in.xy)); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h_tm; \\\n\
    write_imagef(output, coord_in.xy, r_times_h); \\\n\
}\n\
GRUCELL_H_TIMES_R_U8_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_H_TIMES_R_U8_F32_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_H_TIMES_R_F32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_F32_F32toF32_##act_name( \\\n\
    __read_only  image2d_t hstate_in, \\\n\
    __read_only  image2d_t input_r_conv, \\\n\
    __read_only  image2d_t hstate_r_conv, \\\n\
    __write_only image2d_t output, \\\n\
    float input_scale, float input_tail) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h_tm = read_imagef(hstate_in, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    float4 r_times_h = r * h_tm; \\\n\
    write_imagef(output, coord_in.xy, r_times_h); \\\n\
}\n\
\n\
GRUCELL_H_TIMES_R_F32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_H_TIMES_R_F32_F32_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_H_TIMES_R_I32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_h_times_activation_r_I32_F32toI32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    float input_scale, float input_tail) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imagei(hstate_in, coord_in.xy)); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h_tm; \\\n\
    write_imagef(output, coord_in.xy, r_times_h); \\\n\
}\n\
GRUCELL_H_TIMES_R_I32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_H_TIMES_R_I32_F32_F32(HARD_SIGMOID, hard_sigmoid)"; /* end of grucell_h_times_activation_r_cl*/

static const char grucell_reset_after_activation_cl[] = "#define logE        (1.44269502f)\n\
#define twoLogE     (logE * 2.0f)\n\
\n\
float sigmoid(float x)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float hard_sigmoid(float x)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float tanh_func(float x)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define GRUCELL_ACTIVATION_U8_F32_U8(act_name, act_func) \\\n\
__kernel void grucell_reset_after_activation_U8_F32toU8_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imageui(hstate_in, coord_in.xy)); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h1; \\\n\
    float4 h = h0 + r_times_h; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    dst = dst * output_scale + output_zp; \\\n\
    uint4 result = convert_uint4_sat_rte(dst); \\\n\
    write_imageui(output, coord_in.xy, result); \\\n\
    write_imageui(hstate_out, coord_in.xy, result); \\\n\
}\n\
GRUCELL_ACTIVATION_U8_F32_U8(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_ACTIVATION_F32_F32_F32(act_name, act_func) \\\n\
__kernel void grucell_reset_after_activation_F32_F32toF32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h_tm = read_imagef(hstate_in, coord_in.xy); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    float4 r_times_h = r * h1; \\\n\
    float4 h = h0 + r_times_h; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    write_imagef(output, coord_in.xy, dst); \\\n\
    write_imagef(hstate_out, coord_in.xy, dst); \\\n\
}\n\
\n\
GRUCELL_ACTIVATION_F32_F32_F32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define GRUCELL_ACTIVATION_I32_F32_I32(act_name, act_func) \\\n\
__kernel void grucell_reset_after_activation_I32_F32toI32_##act_name( \\\n\
    __read_only  image2d_t        hstate_in, \\\n\
    __read_only  image2d_t        input_z_conv, \\\n\
    __read_only  image2d_t        input_r_conv, \\\n\
    __read_only  image2d_t        input_h_conv, \\\n\
    __read_only  image2d_t        hstate_z_conv, \\\n\
    __read_only  image2d_t        hstate_r_conv, \\\n\
    __read_only  image2d_t        hstate_h_conv, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        hstate_out, \\\n\
    float input_scale, float input_tail, float output_scale, float output_zp) \\\n\
{ \\\n\
    int2 coord_in = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 r0 = read_imagef(input_r_conv, coord_in.xy); \\\n\
    float4 r1 = read_imagef(hstate_r_conv, coord_in.xy); \\\n\
    float4 h1 = read_imagef(hstate_h_conv, coord_in.xy); \\\n\
    float4 h_tm = convert_float4(read_imagei(hstate_in, coord_in.xy)); \\\n\
    float4 h0 = read_imagef(input_h_conv, coord_in.xy); \\\n\
    float4 z0 = read_imagef(input_z_conv, coord_in.xy); \\\n\
    float4 z1 = read_imagef(hstate_z_conv, coord_in.xy); \\\n\
 \\\n\
    float4 r = r0 + r1; \\\n\
    r.x = act_func(r.x); \\\n\
    h_tm = h_tm * input_scale + input_tail; \\\n\
    float4 r_times_h = r * h1; \\\n\
    float4 h = h0 + r_times_h; \\\n\
    float4 z = z0 + z1; \\\n\
    z.x = act_func(z.x); \\\n\
    h = tanh_func(h.x); \\\n\
    float4 dst = (1 - z ) * h + z * h_tm; \\\n\
    dst = dst * output_scale + output_zp; \\\n\
    int4 result = convert_int4_sat_rte(dst); \\\n\
    write_imagei(output, coord_in.xy, result); \\\n\
    write_imagei(hstate_out, coord_in.xy, result); \\\n\
}\n\
GRUCELL_ACTIVATION_I32_F32_I32(SIGMOID, sigmoid)\n\
//GRUCELL_ACTIVATION_U8_F32_U8(HARD_SIGMOID, hard_sigmoid)"; /* end of grucell_reset_after_activation_cl*/

static const char hswish_cl[] = "#define HSWISH_F32_F32_PROCESS() \\\n\
    float4 src, tmp, dst; \\\n\
    src   = read_imagef(input, coord); \\\n\
    tmp   = src + 3; \\\n\
    tmp   = tmp > 0 ? tmp : 0; \\\n\
    tmp   = tmp < 6 ? tmp : 6; \\\n\
    dst   = src * tmp / 6.0f; \\\n\
    write_imagef(output, coord, dst);\n\
\n\
__kernel void hswish_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_F32_F32_PROCESS()\n\
}\n\
\n\
__kernel void hswish_F32toF32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_F32_F32_PROCESS()\n\
}\n\
\n\
\n\
#define HSWISH_U8_U8_PROCESS() \\\n\
    float4 src, tmp, data; \\\n\
    uint4 src0 = read_imageui(input, coord); \\\n\
    src   = convert_float4(src0) * inputScale - inputTail; \\\n\
    tmp   = src + 3; \\\n\
    tmp   = tmp > 0 ? tmp : 0; \\\n\
    tmp   = tmp < 6 ? tmp : 6; \\\n\
    data   = src * tmp / 6.0f; \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
    write_imageui(output, coord, dst);\n\
\n\
__kernel void hswish_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_U8_U8_PROCESS()\n\
}\n\
\n\
__kernel void hswish_U8toU8_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_U8_U8_PROCESS()\n\
}\n\
\n\
\n\
#define HSWISH_I32_I32_PROCESS() \\\n\
    int4 tmp, dst, src; \\\n\
    src   = read_imagei(input, coord); \\\n\
    tmp   = src + 3; \\\n\
    tmp   = tmp > 0 ? tmp : 0; \\\n\
    tmp   = tmp < 6 ? tmp : 6; \\\n\
    dst   = src * tmp / 6; \\\n\
    write_imagei(output, coord, dst);\n\
\n\
__kernel void hswish_I32toI32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    HSWISH_I32_I32_PROCESS()\n\
}\n\
\n\
__kernel void hswish_I32toI32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    HSWISH_I32_I32_PROCESS()\n\
}\n\
"; /* end of hswish_cl*/

static const char instance_normalization_f16_cl[] = "__kernel void instance_norm_meanvari_F16(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum += data.x;\n\
            sqr += data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_meanvari_F16_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum += data.x;\n\
            sqr += data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_F16toF16(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 data, dst;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imagef(input, coord);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_F16toF16_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 data, dst;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imagef(input, coord);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
    }\n\
}\n\
"; /* end of instance_normalization_f16_cl*/

static const char instance_normalization_f32_cl[] = "__kernel void instance_norm_meanvari_F32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum += data.x;\n\
            sqr += data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_meanvari_F32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            data = read_imagef(input, coord);\n\
            coord.y++;\n\
            sum += data.x;\n\
            sqr += data.x * data.x;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_F32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    float4 data, dst;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imagef(input, coord);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_F32toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    bias_val = beta.s0 - scale_vari * mean_vari.s0;\n\
\n\
    float4 data, dst;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imagef(input, coord);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
    }\n\
}\n\
"; /* end of instance_normalization_f32_cl*/

static const char instance_normalization_i32_cl[] = "__kernel void instance_norm_meanvari_I32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    int4 data;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0;\n\
    float e2InScale = input_fl * input_fl;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imagei(input, coord);\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            sqr += (data.x * data.x * e2InScale);\n\
        }\n\
        sum = tmpSum * input_fl;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_meanvari_I32_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    int4 data;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0;\n\
    float e2InScale = input_fl * input_fl;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            data = read_imagei(input, coord);\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            sqr += (data.x * data.x * e2InScale);\n\
        }\n\
        sum = tmpSum * input_fl;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toI32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_fl * output_fl * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_fl;\n\
\n\
    int4 data, dst;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_int4_rte(norm);\n\
        write_imagei(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toI32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_fl * output_fl * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_fl;\n\
\n\
    int4 data, dst;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_int4_rte(norm);\n\
        write_imagei(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_fl * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0);\n\
\n\
    int4 data;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_I32toF32_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = input_fl * scale_vari;\n\
    bias_val = beta.s0 - scale_vari * mean_vari.s0;\n\
\n\
    int4 data;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imagei(input, coord);\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
"; /* end of instance_normalization_i32_cl*/

static const char instance_normalization_u8_cl[] = "__kernel void instance_norm_meanvari_U8(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    uint4 data;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    if(gidx < width)\n\
    {\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imageui(input, coord);\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            tmpSqr += data.x * data.x;\n\
        }\n\
        sqr = (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum = (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_meanvari_U8_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int width,\n\
    int height\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
    int gidy = gidz * height;\n\
\n\
    int2 coord = (int2)(gidx, gidy);\n\
    uint4 data;\n\
    float sum = 0, sqr = 0;\n\
    int tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    int endH = gidy + height;\n\
    if(gidx < width)\n\
    {\n\
        for(; coord.y < endH;)\n\
        {\n\
            data = read_imageui(input, coord);\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            tmpSqr += data.x * data.x;\n\
        }\n\
        sqr = (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum = (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int4 coord_out = (int4)(get_group_id(0) << 2, gidz, 0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 dst = (float4)(0);\n\
        dst.x = sum;\n\
        write_imagef(output, coord_out.xy, dst);\n\
        coord_out.x++;\n\
        dst.x = sqr;\n\
        write_imagef(output, coord_out.xy, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toU8(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data, dst;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
        data.x -= input_zp;\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_uint4_rte(norm);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toU8_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data, dst;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
        data.x -= input_zp;\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        dst = convert_uint4_rte(norm);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toF16(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int4 coord = (int4)(get_global_id(0), 0, gidz, 0);\n\
    int4 coord_para = (int4)(0, gidz, 0, 0);\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data;\n\
    for(coord.y = 0; coord.y < height;coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
        data.x -= input_zp;\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
\n\
__kernel void instance_norm_U8toF16_2D(\n\
    __read_only image2d_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __read_only image2d_t   meanVari,\n\
    __write_only image2d_t  output,\n\
    float eps,\n\
    int rsFlg,\n\
    int input_zp,\n\
    float input_scale,\n\
    float input_fl,\n\
    int output_zp,\n\
    float output_scale,\n\
    float output_fl,\n\
    int width,\n\
    int height,\n\
    float dim_ratio,\n\
    int group_num\n\
    )\n\
{\n\
    int gidz = get_global_id(1);\n\
    int gidy = gidz * height;\n\
    int2 coord = (int2)(get_global_id(0), gidy);\n\
    int2 coord_para = (int2)(0, gidz);\n\
    int endH = gidy + height;\n\
\n\
    float4 gamma = read_imagef(scale, coord_para.yx);\n\
    float4 beta  = read_imagef(bias, coord_para.yx);\n\
    float4 mean_vari = (float4)(0);\n\
    float scale_vari, bias_val;\n\
    float scale_inOut = input_scale * output_scale;\n\
\n\
    for(int i = 0; i < group_num; i++)\n\
    {\n\
        mean_vari.x += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x++;\n\
        mean_vari.y += read_imagef(meanVari, coord_para.xy).x;\n\
        coord_para.x+=3;\n\
    }\n\
    mean_vari *= dim_ratio;\n\
    mean_vari.s1 = mean_vari.s1 - mean_vari.s0 * mean_vari.s0 + eps;\n\
    mean_vari.s1 = rsqrt(mean_vari.s1);\n\
\n\
    scale_vari = gamma.s0 * mean_vari.s1;\n\
    float alpha = scale_inOut * scale_vari;\n\
    bias_val = (beta.s0 - scale_vari * mean_vari.s0) * output_scale + output_zp;\n\
\n\
    uint4 data;\n\
    for(; coord.y < endH; coord.y++)\n\
    {\n\
        data = read_imageui(input, coord);\n\
        data.x -= input_zp;\n\
\n\
        float4 norm;\n\
        norm.x = data.x * alpha + bias_val;\n\
        write_imagef(output, coord, norm);\n\
    }\n\
}\n\
"; /* end of instance_normalization_u8_cl*/

static const char l2normalizescale_axis0_cl[] = "\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void l2normalizescale_axis0_F32_F32toF32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int2 coord_scale = (int2)(gidx, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        src = read_imagef(input, coord);\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for(coord.x = gidx; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        src         = read_imagef(input, coord);\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void l2normalizescale_axis0_U8_F32toU8_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int gidx = get_global_id(0);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int2 coord_scale = (int2)(gidx, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for(coord.x = gidx; coord.x < axis_size; coord.x += 16)\n\
    {\n\
        src         = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
\n\
\n\
"; /* end of l2normalizescale_axis0_cl*/

static const char l2normalizescale_axis1_cl[] = "\n\
__kernel __attribute__((reqd_work_group_size(1, 16, 1))) void l2normalizescale_axis1_F32_F32toF32_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps\n\
    )\n\
{\n\
    int lidx = get_local_id(1);\n\
    int gidy = get_global_id(1);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(get_global_id(0), gidy );\n\
    int2 coord_scale = (int2)(gidy, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        src = read_imagef(input, coord);\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for(coord.y = gidy; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        src         = read_imagef(input, coord);\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        write_imagef(output, coord, result.xxxx);\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(1, 16, 1))) void l2normalizescale_axis1_U8_F32toU8_2D(\n\
    __read_only  image2d_t input,\n\
    __read_only  image2d_t scale,\n\
    __write_only image2d_t output,\n\
                       int axis,\n\
                       int axis_size,\n\
                     float rsEps,\n\
                     float inputScale,\n\
                     float inputTail,\n\
                     float outputScale,\n\
                     float outputZP\n\
    )\n\
{\n\
    int lidx = get_local_id(1);\n\
    int gidy = get_global_id(1);\n\
    float4 src, scale_value, result;\n\
    float sum  = 0.0f, pSum = 0.0f, rsqrt_sum = 0.0f;\n\
    int2 coord = (int2)(get_global_id(0), gidy );\n\
    int2 coord_scale = (int2)(gidy, 0);\n\
    __local float lcl_sum[16];\n\
    for(; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        pSum += (src.x * src.x);\n\
    }\n\
    lcl_sum[lidx] = pSum;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    float4 *pLocalPtr = (float4 *)&lcl_sum[0];\n\
    float4 one = (float4)(1, 1, 1, 1);\n\
    float4 data0;\n\
    data0 = pLocalPtr[0] + pLocalPtr[1] + pLocalPtr[2] + pLocalPtr[3];\n\
    sum = dot(data0, one);\n\
    rsqrt_sum = (sum == 0 ? rsEps : rsqrt(sum));\n\
    for(coord.y = gidy; coord.y < axis_size; coord.y += 16)\n\
    {\n\
        src         = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
        scale_value = read_imagef(scale, coord_scale);\n\
        result      = src * rsqrt_sum * scale_value;\n\
        uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
        write_imageui(output, coord, dst);\n\
    }\n\
}\n\
"; /* end of l2normalizescale_axis1_cl*/

static const char layer_normalization_cl[] = "\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layer_norm_F32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float e2InScale,\n\
    float scale_inOut,\n\
    float sumZpScale,\n\
    float zp2ScaleE2,\n\
    float sumZpScaleE2,\n\
    int width,\n\
    int height,\n\
    float dim_ratio\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(lidx, get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 data, dst;\n\
    float2 sumSqr = (float2)(0);\n\
    float scale_vari, bias_val;\n\
    __local float2 local_sum[16];\n\
\n\
    for(; coord.x < width;)\n\
    {\n\
        data = read_imagef(input, coord);\n\
        coord.x += 16;\n\
        sumSqr.x += data.x;\n\
        sumSqr.y += data.x * data.x;\n\
    }\n\
    local_sum[lidx] = sumSqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    if(lidx == 0)\n\
    {\n\
        for(int i = 1; i < 16; i++)\n\
        {\n\
            sumSqr += local_sum[i];\n\
        }\n\
        local_sum[0] = sumSqr;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    sumSqr = local_sum[0] * dim_ratio;\n\
    sumSqr.s1 = sumSqr.s1 - sumSqr.s0 * sumSqr.s0 + eps;\n\
    sumSqr.s1 = rsqrt(sumSqr.s1);\n\
\n\
    for(coord.x = lidx; coord.x < width;)\n\
    {\n\
        float4 gamma = read_imagef(scale, coord.xw);\n\
        float4 beta  = read_imagef(bias, coord.xw);\n\
        data = read_imagef(input, coord);\n\
\n\
        scale_vari = gamma.s0 * sumSqr.s1;\n\
        bias_val = (beta.s0 - scale_vari * sumSqr.s0);\n\
\n\
        dst.x = data.x * scale_vari + bias_val;\n\
        write_imagef(output, coord, dst);\n\
        coord.x += 16;\n\
    }\n\
}\n\
\n\
__kernel __attribute__((reqd_work_group_size(16, 1, 1))) void layer_norm_U8toU8(\n\
    __read_only image2d_array_t   input,\n\
    __read_only image2d_t   bias,\n\
    __read_only image2d_t   scale,\n\
    __write_only image2d_array_t  output,\n\
    float eps,\n\
    float input_zp,\n\
    float input_scale,\n\
    float output_zp,\n\
    float output_scale,\n\
    float e2InScale,\n\
    float scale_inOut,\n\
    float sumZpScale,\n\
    float zp2ScaleE2,\n\
    float sumZpScaleE2,\n\
    int width,\n\
    int height,\n\
    float dim_ratio\n\
    )\n\
{\n\
    int lidx = get_local_id(0);\n\
    int4 coord = (int4)(lidx, get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 data, dst;\n\
    float2 sumSqr;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float scale_vari, bias_val;\n\
    __local uint local_sum[1];\n\
    __local uint local_sqr[1];\n\
\n\
    if(lidx == 0)\n\
    {\n\
        local_sum[0] = 0;\n\
        local_sqr[0] = 0;\n\
    }\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    for(; coord.x < width;)\n\
    {\n\
        data = read_imageui(input, coord);\n\
        coord.x+=16;\n\
        tmpSum += data.x;\n\
        tmpSqr += data.x * data.x;\n\
    }\n\
    atom_add(local_sum, tmpSum);\n\
    atom_add(local_sqr, tmpSqr);\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
    tmpSum = local_sum[0];\n\
    tmpSqr = local_sqr[0];\n\
    //sumSqr.x = ((float)tmpSum - width * input_zp) * input_scale;\n\
    //sumSqr.y = ((float)tmpSqr - 2 * input_zp * (float)tmpSum + width * input_zp * input_zp) * e2InScale;\n\
    sumSqr.x = (float)tmpSum * input_scale - sumZpScale;\n\
    sumSqr.y = (float)tmpSqr * e2InScale - zp2ScaleE2 * (float)tmpSum + sumZpScaleE2;\n\
\n\
    sumSqr *= dim_ratio;\n\
    sumSqr.s1 = sumSqr.s1 - sumSqr.s0 * sumSqr.s0 + eps;\n\
    sumSqr.s1 = rsqrt(sumSqr.s1);\n\
\n\
    for(coord.x = lidx; coord.x < width;)\n\
    {\n\
        float4 gamma = read_imagef(scale, coord.xw);\n\
        float4 beta  = read_imagef(bias, coord.xw);\n\
        data = read_imageui(input, coord);\n\
\n\
        scale_vari = gamma.s0 * sumSqr.s1;\n\
        float alpha = scale_inOut * scale_vari;\n\
        bias_val = (beta.s0 - scale_vari * sumSqr.s0) * output_scale + output_zp;\n\
\n\
        float tmpVal = data.x - input_zp;\n\
\n\
        float4 norm;\n\
        norm.x = tmpVal * alpha + bias_val;\n\
        dst = convert_uint4_rte(norm);\n\
        write_imageui(output, coord, dst);\n\
        coord.x+=16;\n\
    }\n\
}\n\
"; /* end of layer_normalization_cl*/

static const char log_softmax_axis0_cl[] = "#define rlogE    (0.693147182f)\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_axis0_F32toF32\n\
    (\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
                            int   axis,\n\
                            float beta,\n\
                            float scale,\n\
                            float scaleOut,\n\
                            float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int width = get_image_width(input);\n\
    int4 coord_in = (int4)(0, y, z, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_F32toF32_2D\n\
    (\n\
    __read_only   image2d_t input,\n\
    __write_only  image2d_t output,\n\
                      int   axis,\n\
                      float beta,\n\
                      float scale,\n\
                      float scaleOut,\n\
                      float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int width = get_image_width(input);\n\
    int2 coord_in = (int2)(0, y);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.0f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           int   axis,\n\
                           float beta,\n\
                           float scale,\n\
                           float scaleOut,\n\
                           float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int width = get_image_width(input);\n\
    int4 coord_in = (int4)(0, y, z, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis0_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     int   axis,\n\
                     float beta,\n\
                     float scale,\n\
                     float scaleOut,\n\
                     float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int width = get_image_width(input);\n\
    int2 coord_in = (int2)(0, y);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.x = 1; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.x++;\n\
\n\
        sum += exp2((src.x - maxValue.x)*scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.x = 0; coord_in.x < width; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.x++;\n\
    }\n\
}\n\
#undef rlogE\n\
"; /* end of log_softmax_axis0_cl*/

static const char log_softmax_axis1_cl[] = "#define rlogE    (0.693147182f)\n\
\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_axis1_F32toF32\n\
    (\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
                            int   axis,\n\
                            float beta,\n\
                            float scale,\n\
                            float scaleOut,\n\
                            float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int height = get_image_height(input);\n\
    int4 coord_in = (int4)(x, 0, z, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_F32toF32_2D\n\
    (\n\
    __read_only   image2d_t input,\n\
    __write_only  image2d_t output,\n\
                      int   axis,\n\
                      float beta,\n\
                      float scale,\n\
                      float scaleOut,\n\
                      float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int height = get_image_height(input);\n\
    int2 coord_in = (int2)(x, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.0f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = 1.0f * LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_array_t output,\n\
                           int   axis,\n\
                           float beta,\n\
                           float scale,\n\
                           float scaleOut,\n\
                           float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int height = get_image_height(input);\n\
    int4 coord_in = (int4)(x, 0, z, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis1_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                     int   axis,\n\
                     float beta,\n\
                     float scale,\n\
                     float scaleOut,\n\
                     float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int height = get_image_height(input);\n\
    int2 coord_in = (int2)(x, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.y = 1; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.y++;\n\
\n\
        sum += exp2((src.x - maxValue.x)*scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.y = 0; coord_in.y < height; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.y++;\n\
    }\n\
}\n\
#undef rlogE\n\
"; /* end of log_softmax_axis1_cl*/

static const char log_softmax_axis2_cl[] = "#define rlogE    (0.693147182f)\n\
float LOG(float x)\n\
{\n\
    x = log2(x);\n\
    return x * rlogE;\n\
}\n\
\n\
__kernel void log_softmax_axis2_F32toF32\n\
    (\n\
    __read_only   image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
                            int   axis,\n\
                            float beta,\n\
                            float scale,\n\
                            float scaleOut,\n\
                            float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int depth = get_image_array_size(input);\n\
    int4 coord_in = (int4)(x, y, 0, 0);\n\
    float4 maxValue;\n\
    float4 src, dst = {0.0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = read_imagef(input, coord_in);\n\
    for (coord_in.z = 1; coord_in.z < depth; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.z++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
        coord_in.z++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = read_imagef(input, coord_in);\n\
\n\
        dst.x = (src.x - maxValue.x) * beta - logSum;\n\
        write_imagef(output, coord_in, dst);\n\
        coord_in.z++;\n\
    }\n\
}\n\
\n\
__kernel void log_softmax_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only  image2d_array_t output,\n\
                            int   axis,\n\
                            float beta,\n\
                            float scale,\n\
                            float scaleOut,\n\
                            float zpOut\n\
    )\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int depth = get_image_array_size(input);\n\
    int4 coord_in = (int4)(x, y, 0, 0);\n\
    float4 maxValue;\n\
    float4 src;\n\
    uint4 dst = {0};\n\
\n\
    // Find max element value which we'll use to ensure numerical stability\n\
    // taking advantage of the following equality:\n\
    // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C))\n\
    maxValue = convert_float4(read_imageui(input, coord_in));\n\
    for (coord_in.z = 1; coord_in.z < depth; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.z++;\n\
\n\
        maxValue = maxValue > src ? maxValue : src;\n\
    }\n\
\n\
    // Compute sum.\n\
    float sum = 0.f;\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
        coord_in.z++;\n\
\n\
        sum += exp2((src.x - maxValue.x) * scale);\n\
    }\n\
\n\
    // Compute result.\n\
    float logSum = LOG(sum);\n\
    for (coord_in.z = 0; coord_in.z < depth; )\n\
    {\n\
        src = convert_float4(read_imageui(input, coord_in));\n\
\n\
        dst.x = convert_uint(((src.x - maxValue.x) * beta - logSum) * scaleOut + zpOut);\n\
\n\
        write_imageui(output, coord_in, dst);\n\
        coord_in.z++;\n\
    }\n\
}\n\
#undef rlogE\n\
"; /* end of log_softmax_axis2_cl*/

static const char logical_not_cl[] = "__kernel void logical_not_I8toI8(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_array_t  output)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 src   = read_imagei(input, coord);\n\
    int4 dst   = !src;\n\
    dst.x = dst.x & 1;\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void logical_not_I8toI8_2D(\n\
    __read_only image2d_t   input,\n\
    __write_only image2d_t  output)\n\
{\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1));\n\
    int4 src   = read_imagei(input, coord);\n\
    int4 dst   = !src;\n\
    dst.x = dst.x & 1;\n\
    write_imagei(output, coord, dst);\n\
}\n\
"; /* end of logical_not_cl*/

static const char logical_ops_cl[] = "#define TENSORLOGICAL(name, lgc_op, lgc_op2) \\\n\
__kernel void logical_##name##_I8toI8( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __read_only image2d_array_t   input1, \\\n\
    __write_only image2d_array_t  output) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 src0; \\\n\
    int4 src1; \\\n\
    READ_IMAGEI_2DARRAY(src0, input, coord); \\\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord); \\\n\
    int4 dst  = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    dst.x = dst.x & 1; \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
TENSORLOGICAL(or,  ||, )\n\
TENSORLOGICAL(and, &&, )\n\
TENSORLOGICAL(xor, ^,  !!)\n\
\n\
\n\
#define TENSORLOGICAL_2D(name, lgc_op, lgc_op2) \\\n\
__kernel void logical_##name##_I8toI8_2D( \\\n\
    __read_only image2d_t   input, \\\n\
    __read_only image2d_t   input1, \\\n\
    __write_only image2d_t  output) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int4 src0 = read_imagei(input, coord); \\\n\
    int4 src1 = read_imagei(input1, coord); \\\n\
    int4 dst  = (lgc_op2(src0))lgc_op(lgc_op2(src1)); \\\n\
    dst.x = dst.x & 1; \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
\n\
TENSORLOGICAL_2D(or,  ||, )\n\
TENSORLOGICAL_2D(and, &&, )\n\
TENSORLOGICAL_2D(xor, ^,  !!)\n\
"; /* end of logical_ops_cl*/

static const char lstmunit_activation_BP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_BP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_BP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_F32_cl*/

static const char lstmunit_activation_BP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_BP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_BP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_BP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_BP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_BP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_BP_U8_cl*/

static const char lstmunit_activation_B_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_B_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_B_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = read_imagef(input_i_conv, coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_F32TOU8(HARD_SIGMOID, hard_sigmoid)"; /* end of lstmunit_activation_B_F32_cl*/

static const char lstmunit_activation_B_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_B_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_B_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_B_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_i, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = data_i_t + b0; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_B_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_B_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_B_U8_cl*/

static const char lstmunit_activation_CBP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CBP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CBP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_F32_cl*/

static const char lstmunit_activation_CBP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CBP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CBP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CBP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CBP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CBP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CBP_U8_cl*/

static const char lstmunit_activation_CB_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_F32_cl*/

static const char lstmunit_activation_CB_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CB_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CB_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = data_f_t + b1; \\\n\
    data_g_t = data_g_t + b2; \\\n\
    data_o_t = data_o_t + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CB_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CB_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CB_U8_cl*/

static const char lstmunit_activation_CLP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CLP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CLP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CLP_F32_cl*/

static const char lstmunit_activation_CLP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CLP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CLP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CLP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CLP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CLP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CLP_U8_cl*/

static const char lstmunit_activation_CL_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = read_imagef(input_f_conv, coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv, coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv, coord_in.xy); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CL_F32_cl*/

static const char lstmunit_activation_CL_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CL_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CL_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  bias_f, \\\n\
    __read_only  image2d_t  bias_c, \\\n\
    __read_only  image2d_t  bias_o, \\\n\
    __read_only  image2d_t  layer_norm_wf, \\\n\
    __read_only  image2d_t  layer_norm_wc, \\\n\
    __read_only  image2d_t  layer_norm_wo, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b1, b2, b3; \\\n\
    float4 w1, w2, w3; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CL_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CL_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CL_U8_cl*/

static const char lstmunit_activation_CSP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CSP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CSP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_F32_cl*/

static const char lstmunit_activation_CSP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CSP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CSP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CSP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CSP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CSP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_CSP_U8_cl*/

static const char lstmunit_activation_CS_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CS_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
#define LSTM_ACTIVATION_CS_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
"; /* end of lstmunit_activation_CS_F32_cl*/

static const char lstmunit_activation_CS_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_CS_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_CS_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_CS_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src1, src2, src3; \\\n\
    float4  src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = 1.0f - data_f_t; \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_CS_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_CS_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
"; /* end of lstmunit_activation_CS_U8_cl*/

static const char lstmunit_activation_LP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_LP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_LP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t        input_i_conv, \\\n\
    __read_only  image2d_t        input_f_conv, \\\n\
    __read_only  image2d_t        input_c_conv, \\\n\
    __read_only  image2d_t        input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 w0, w1, w2, w3; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 * w0 + b0; \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_LP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_LP_F32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_LP_F32_cl*/

static const char lstmunit_activation_L_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_L_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_L_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t        input_i_conv, \\\n\
    __read_only  image2d_t        input_f_conv, \\\n\
    __read_only  image2d_t        input_c_conv, \\\n\
    __read_only  image2d_t        input_o_conv, \\\n\
    __read_only  image2d_t        cell_state_in, \\\n\
    __read_only  image2d_t        bias_i, \\\n\
    __read_only  image2d_t        bias_f, \\\n\
    __read_only  image2d_t        bias_c, \\\n\
    __read_only  image2d_t        bias_o, \\\n\
    __read_only  image2d_t        layer_norm_wi, \\\n\
    __read_only  image2d_t        layer_norm_wf, \\\n\
    __read_only  image2d_t        layer_norm_wc, \\\n\
    __read_only  image2d_t        layer_norm_wo, \\\n\
    __write_only image2d_t        output, \\\n\
    __write_only image2d_t        cell_state_out, \\\n\
    __write_only image2d_t        h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    float4 b0, b1, b2, b3; \\\n\
    float4 w0, w1, w2, w3; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    w0 = read_imagef(layer_norm_wi, coord_in.xw); \\\n\
    w1 = read_imagef(layer_norm_wf, coord_in.xw); \\\n\
    w2 = read_imagef(layer_norm_wc, coord_in.xw); \\\n\
    w3 = read_imagef(layer_norm_wo, coord_in.xw); \\\n\
    b0 = read_imagef(bias_i, coord_in.xw); \\\n\
    b1 = read_imagef(bias_f, coord_in.xw); \\\n\
    b2 = read_imagef(bias_c, coord_in.xw); \\\n\
    b3 = read_imagef(bias_o, coord_in.xw); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 * w0 + b0; \\\n\
    data_f_t = src1 * w1 + b1; \\\n\
    data_g_t = src2 * w2 + b2; \\\n\
    data_o_t = src3 * w3 + b3; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_L_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_L_F32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_L_F32_cl*/

static const char lstmunit_activation_SP_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_F32_cl*/

static const char lstmunit_activation_SP_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_SP_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_SP_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_SP_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_SP_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_SP_U8_cl*/

static const char lstmunit_activation_S_F32_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_S_F32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_F32toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_F32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_F32(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_S_F32TOU8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_F32toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = read_imagef(input_i_conv,  coord_in.xy); \\\n\
    src1  = read_imagef(input_f_conv,  coord_in.xy); \\\n\
    src2  = read_imagef(input_c_conv,  coord_in.xy); \\\n\
    src3  = read_imagef(input_o_conv,  coord_in.xy); \\\n\
    src10 = read_imagef(hstate_i_conv, coord_in.xy); \\\n\
    src11 = read_imagef(hstate_f_conv, coord_in.xy); \\\n\
    src12 = read_imagef(hstate_c_conv, coord_in.xy); \\\n\
    src13 = read_imagef(hstate_o_conv, coord_in.xy); \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_F32TOU8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_F32TOU8(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_F32_cl*/

static const char lstmunit_activation_S_U8_cl[] = "float4 sigmoid(float4 x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
float4 hard_sigmoid(float4 x, float logE)\n\
{\n\
    x = 0.2 * x + 0.5;\n\
    x = clamp(x, 0, 1);\n\
    return x;\n\
}\n\
float4 tangentH(float4 x, float twoLogE)\n\
{\n\
    x *= -twoLogE;\n\
    x = 1 + exp2(x);\n\
    x = 1 / x;\n\
    return 2 * x - 1;\n\
}\n\
\n\
\n\
#define LSTM_ACTIVATION_S_U8(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_U8toU8_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t * out_scale + out_zp; \\\n\
    uint4 data_o_u = convert_uint4_sat_rte(data_o_t); \\\n\
    write_imageui(output, coord_in.zy, data_o_u); \\\n\
    write_imageui(h_state_out, coord_in.zy, data_o_u); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_U8(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_U8(HARD_SIGMOID, hard_sigmoid)\n\
\n\
\n\
#define LSTM_ACTIVATION_S_U8TOF32(act_name, act_func) \\\n\
__kernel void lstmunit_activation_S_U8toF32_F32_##act_name( \\\n\
    __read_only  image2d_t  input_i_conv, \\\n\
    __read_only  image2d_t  input_f_conv, \\\n\
    __read_only  image2d_t  input_c_conv, \\\n\
    __read_only  image2d_t  input_o_conv, \\\n\
    __read_only  image2d_t  cell_state_in, \\\n\
    __read_only  image2d_t  hstate_i_conv, \\\n\
    __read_only  image2d_t  hstate_f_conv, \\\n\
    __read_only  image2d_t  hstate_c_conv, \\\n\
    __read_only  image2d_t  hstate_o_conv, \\\n\
    __write_only image2d_t  output, \\\n\
    __write_only image2d_t  cell_state_out, \\\n\
    __write_only image2d_t  h_state_out, \\\n\
    float logE, float twoLogE, float forget_bias, float clip_Max_F, float clip_Min_F, \\\n\
    float in_fc_i_scale,  float in_fc_i_tail,  float in_fc_f_scale,  float in_fc_f_tail, \\\n\
    float in_fc_c_scale,  float in_fc_c_tail,  float in_fc_o_scale,  float in_fc_o_tail, \\\n\
    float hstate_i_scale, float hstate_i_tail, float hstate_f_scale, float hstate_f_tail, \\\n\
    float hstate_c_scale, float hstate_c_tail, float hstate_o_scale, float hstate_o_tail, \\\n\
    float out_scale, float out_zp) \\\n\
{ \\\n\
    int4 coord_in = (int4)(get_global_id(0), get_global_id(1), get_global_id(0), 0); \\\n\
    float4  src0, src1, src2, src3; \\\n\
    float4  src10, src11, src12, src13; \\\n\
    float4 data_i_t, data_f_t, data_g_t, data_o_t, data_c_t; \\\n\
    src0  = convert_float4(read_imageui(input_i_conv, coord_in.xy)) * in_fc_i_scale + in_fc_i_tail; \\\n\
    src10 = convert_float4(read_imageui(hstate_i_conv, coord_in.xy)) * hstate_i_scale + hstate_i_tail; \\\n\
    src1  = convert_float4(read_imageui(input_f_conv, coord_in.xy)) * in_fc_f_scale + in_fc_f_tail; \\\n\
    src11 = convert_float4(read_imageui(hstate_f_conv, coord_in.xy)) * hstate_f_scale + hstate_f_tail; \\\n\
    src2  = convert_float4(read_imageui(input_c_conv, coord_in.xy)) * in_fc_c_scale + in_fc_c_tail; \\\n\
    src12 = convert_float4(read_imageui(hstate_c_conv, coord_in.xy)) * hstate_c_scale + hstate_c_tail; \\\n\
    src3  = convert_float4(read_imageui(input_o_conv, coord_in.xy)) * in_fc_o_scale + in_fc_o_tail; \\\n\
    src13 = convert_float4(read_imageui(hstate_o_conv, coord_in.xy)) * hstate_o_scale + hstate_o_tail; \\\n\
    data_c_t = read_imagef(cell_state_in, coord_in.xy); \\\n\
    data_i_t = src0 + src10; \\\n\
    data_f_t = src1 + src11; \\\n\
    data_g_t = src2 + src12; \\\n\
    data_o_t = src3 + src13; \\\n\
    data_i_t = act_func(data_i_t, logE); \\\n\
    data_f_t = act_func(data_f_t + forget_bias, logE); \\\n\
    data_g_t = tangentH(data_g_t, twoLogE); \\\n\
    data_i_t = data_i_t * data_g_t; \\\n\
    data_c_t = data_c_t * data_f_t + data_i_t; \\\n\
    data_o_t = act_func(data_o_t, logE); \\\n\
    data_c_t = data_c_t > clip_Max_F ? clip_Max_F : data_c_t; \\\n\
    data_c_t = data_c_t < clip_Min_F ? clip_Min_F : data_c_t; \\\n\
    write_imagef(cell_state_out, coord_in.zy, data_c_t); \\\n\
    data_c_t = tangentH(data_c_t, twoLogE); \\\n\
    data_o_t = data_o_t * data_c_t; \\\n\
    write_imagef(output, coord_in.zy, data_o_t); \\\n\
    write_imagef(h_state_out, coord_in.zy, data_o_t); \\\n\
}\n\
\n\
LSTM_ACTIVATION_S_U8TOF32(SIGMOID, sigmoid)\n\
LSTM_ACTIVATION_S_U8TOF32(HARD_SIGMOID, hard_sigmoid)\n\
"; /* end of lstmunit_activation_S_U8_cl*/

static const char matrixmul_cl[] = "__kernel void gemm_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.zy);\n\
        tempB0 = read_imagef(inputB, coord.xz);\n\
        coord.z++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_F32F32toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(get_global_id(0), 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.x < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_a.x++;\n\
        coord_b.y++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_b.y = get_global_id(1);\n\
    coord_b.z = get_global_id(2);\n\
    write_imagef(output, coord_b, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.zy);\n\
        tempB0 = read_imagef(inputB, coord.zx);\n\
        coord.z++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32F32toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, get_global_id(0), (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.x < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_a.x++;\n\
        coord_b.x++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_a.x = get_global_id(0);\n\
    coord_a.z = get_global_id(2);\n\
    write_imagef(output, coord_a, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32I8toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.zy);\n\
        tempB0 = convert_float4(read_imagei(inputB, coord.zx));\n\
        coord.z++;\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_transb_F32I8toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord_a = (int4)(0, get_global_id(1), (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(0, get_global_id(0), (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.x < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = convert_float4(read_imagei(inputB, coord_b));\n\
        tempB0.x = (tempB0.x - zp_b) * scale_b;\n\
        coord_a.x++;\n\
        coord_b.x++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_a.x = get_global_id(0);\n\
    coord_a.z = get_global_id(2);\n\
    write_imagef(output, coord_a, sum);\n\
}\n\
"; /* end of matrixmul_cl*/

static const char matrixmul_transA_cl[] = "__kernel void gemm_transa_F32F32toF32_2D(\n\
    __read_only image2d_t   inputA,\n\
    __read_only image2d_t   inputB,\n\
    __write_only image2d_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord.z < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord.yz);\n\
        tempB0 = read_imagef(inputB, coord.xz);\n\
        coord.z++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
    write_imagef(output, coord.xy, sum);\n\
}\n\
\n\
__kernel void gemm_transa_F32F32toF32_3D(\n\
    __read_only image2d_array_t   inputA,\n\
    __read_only image2d_array_t   inputB,\n\
    __write_only image2d_array_t  output,\n\
    int M,\n\
    int K,\n\
    int N,\n\
    int ac2zero,\n\
    int bc2zero,\n\
    float scale_a,\n\
    float zp_a,\n\
    float scale_b,\n\
    float zp_b,\n\
    float scale_out,\n\
    float zp_out\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int4 coord_a = (int4)(gidy, 0, (ac2zero ? 0 : get_global_id(2)), 0);\n\
    int4 coord_b = (int4)(gidx, 0, (bc2zero ? 0 : get_global_id(2)), 0);\n\
\n\
    float4 sum = (float4)(0);\n\
\n\
    for(; coord_a.y < K;)\n\
    {\n\
        float4 tempA0;\n\
        float4 tempB0;\n\
\n\
        tempA0 = read_imagef(inputA, coord_a);\n\
        tempB0 = read_imagef(inputB, coord_b);\n\
        coord_a.y++;\n\
        coord_b.y++;\n\
\n\
        sum = sum + tempA0 * tempB0;\n\
    }\n\
\n\
    coord_b.y = gidy;\n\
    coord_b.z = get_global_id(2);\n\
    write_imagef(output, coord_b, sum);\n\
}\n\
"; /* end of matrixmul_transA_cl*/

static const char maximum_cl[] = "__kernel void maximum_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4 dst = src0 > src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 dst = src0 > src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 src0;\n\
    uint4 src1;\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 > data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    uint4 src0 = read_imageui(input0, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 > data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
\n\
__kernel void maximum_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
\n\
    int4 dst = src0 > src1 ? src0 : src1;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void maximum_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    int4 src0 = read_imagei(input0, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
\n\
    int4 dst = src0 > src1 ? src0 : src1;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
"; /* end of maximum_cl*/

static const char minimum_cl[] = "__kernel void minimum_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4 dst = src0 < src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 dst = src0 < src1 ? src0 : src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 src0;\n\
    uint4 src1;\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 < data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    uint4 src0 = read_imageui(input0, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
    float4 data = data0 < data1 ? data0 : data1;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
\n\
__kernel void minimum_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
\n\
    int4 dst = src0 < src1 ? src0 : src1;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void minimum_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    int4 src0 = read_imagei(input0, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
\n\
    int4 dst = src0 < src1 ? src0 : src1;\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
"; /* end of minimum_cl*/

static const char moments_axis0_cl[] = "__kernel void moments_axis0_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(0, gidy, gidz, 0);\n\
    uint data;\n\
    float sum = 0, sqr = 0;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    {\n\
        for(coord0.x = 0; coord0.x < width;)\n\
        {\n\
            data = read_imageui(input, coord0).x;\n\
            coord0.x++;\n\
            tmpSum += (data);\n\
            tmpSqr += (data * data);\n\
        }\n\
        sqr = convert_float(as_int(tmpSqr - 2 * input_zp * tmpSum + width * input_zp * input_zp)) * e2InScale;\n\
        sum = convert_float(as_int(tmpSum - width * input_zp)) * input_scale;\n\
    }\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
#define MOMENTS_AXIS0_F(src0_type_name) \\\n\
__kernel void moments_axis0_##src0_type_name##to##src0_type_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_t  output_mean, \\\n\
    __write_only image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidy = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
 \\\n\
    int4 coord0 = (int4)(0, gidy, gidz, 0); \\\n\
    float data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    for(coord0.x = 0; coord0.x < width;) \\\n\
    { \\\n\
        data = read_imagef(input, coord0).x; \\\n\
        coord0.x++; \\\n\
        sum += (data); \\\n\
        sqr += (data * data); \\\n\
    } \\\n\
 \\\n\
    float4 mean, vari; \\\n\
    mean.x = sum * dimRatio; \\\n\
    vari.x = sqr * dimRatio; \\\n\
    vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidy, gidz); \\\n\
    write_imagef(output_mean, coord_out, mean); \\\n\
    write_imagef(output_vari, coord_out, vari); \\\n\
}\n\
MOMENTS_AXIS0_F(F32)\n\
\n\
__kernel void moments_axis0_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidy = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(0, gidy, gidz, 0);\n\
    float data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.x = 0; coord0.x < width;)\n\
    {\n\
        data = convert_float(read_imagei(input, coord0).x);\n\
        coord0.x++;\n\
\n\
        sum = sum + data;\n\
        sqr = sqr + data * data;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio * input_scale;\n\
    vari.x = sqr * dimRatio * input_scale * input_scale;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidy, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}"; /* end of moments_axis0_cl*/

static const char moments_axis01_cl[] = "__kernel void moments_axis01_U8toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    uint4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        int tmpSum = 0, tmpSqr = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = read_imageui(input, coord);\n\
            coord.y++;\n\
            tmpSum += data.x;\n\
            tmpSqr += data.x * data.x;\n\
        }\n\
        sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum += (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
\n\
#define MOMENTS_AXIS01_F(src0_type_name) \\\n\
__kernel void moments_axis01_##src0_type_name##to##src0_type_name( \\\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
    int lidx = get_local_id(0); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, 0, gidz, 0); \\\n\
    float4 data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16) \\\n\
    { \\\n\
        for(coord.y = 0; coord.y < height;) \\\n\
        { \\\n\
            data = read_imagef(input, coord); \\\n\
            coord.y++; \\\n\
            sum += data.x; \\\n\
            sqr += data.x * data.x; \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int2 coord_out = (int2)(gidz, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = 0; sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        write_imagef(output_mean, coord_out, mean); \\\n\
        write_imagef(output_vari, coord_out, vari); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS01_F(F32)\n\
\n\
__kernel void moments_axis01_I32toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, gidz, 0);\n\
    float4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
    {\n\
        float tmpSum = 0, tmpSqr = 0;\n\
        for(coord.y = 0; coord.y < height;)\n\
        {\n\
            data = convert_float4(read_imagei(input, coord));\n\
            coord.y++;\n\
\n\
            tmpSum = tmpSum + data.x;\n\
            tmpSqr = tmpSqr + data.x * data.x;\n\
        }\n\
        sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
        sum += (tmpSum - height * input_zp) * input_scale;\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(gidz, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
"; /* end of moments_axis01_cl*/

static const char moments_axis012_cl[] = "__kernel void moments_axis012_U8toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    uint4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.z = 0; coord.z < chn; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
        {\n\
            int tmpSum = 0, tmpSqr = 0;\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                data = read_imageui(input, coord);\n\
                coord.y++;\n\
                tmpSum += data.x;\n\
                tmpSqr += data.x * data.x;\n\
            }\n\
            sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
            sum += (tmpSum - height * input_zp) * input_scale;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
\n\
#define MOMENTS_AXIS012_F(src0_type_name) \\\n\
__kernel void moments_axis012_##src0_type_name##to##src0_type_name( \\\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int lidx = get_local_id(0); \\\n\
 \\\n\
    int4 coord = (int4)(gidx, 0, 0, 0); \\\n\
    float4 data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    __local float lcl_sum[16]; \\\n\
    __local float lcl_sqr[16]; \\\n\
 \\\n\
    for(coord.z = 0; coord.z < chn; coord.z++) \\\n\
    { \\\n\
        for(coord.x = gidx; coord.x < width; coord.x += 16) \\\n\
        { \\\n\
            for(coord.y = 0; coord.y < height;) \\\n\
            { \\\n\
                data = read_imagef(input, coord); \\\n\
                coord.y++; \\\n\
                sum += data.x; \\\n\
                sqr += data.x * data.x; \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
    lcl_sum[lidx] = sum; \\\n\
    lcl_sqr[lidx] = sqr; \\\n\
    barrier(CLK_LOCAL_MEM_FENCE); \\\n\
 \\\n\
    int2 coord_out = (int2)(0, 0); \\\n\
    if(lidx == 0) \\\n\
    { \\\n\
        float4 one = (float4)(1, 1, 1, 1); \\\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum; \\\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr; \\\n\
 \\\n\
        sum = 0; sqr = 0; \\\n\
        for(int i = 0; i < 4; i++) \\\n\
        { \\\n\
            sum += dot(tmp_sum[i], one); \\\n\
            sqr += dot(tmp_sqr[i], one); \\\n\
        } \\\n\
 \\\n\
        float4 mean, vari; \\\n\
        mean.x = sum * dimRatio; \\\n\
        vari.x = sqr * dimRatio; \\\n\
        vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
        write_imagef(output_mean, coord_out, mean); \\\n\
        write_imagef(output_vari, coord_out, vari); \\\n\
    } \\\n\
}\n\
MOMENTS_AXIS012_F(F32)\n\
\n\
__kernel void moments_axis012_I32toF32(\n\
    image2d_array_t   input, image2d_t  output_mean, image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int lidx = get_local_id(0);\n\
\n\
    int4 coord = (int4)(gidx, 0, 0, 0);\n\
    int4 data;\n\
    float sum = 0, sqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    __local float lcl_sum[16];\n\
    __local float lcl_sqr[16];\n\
\n\
    for(coord.z = 0; coord.z < chn; coord.z++)\n\
    {\n\
        for(coord.x = gidx; coord.x < width; coord.x += 16)\n\
        {\n\
            int tmpSum = 0, tmpSqr = 0;\n\
            for(coord.y = 0; coord.y < height;)\n\
            {\n\
                data = read_imagei(input, coord);\n\
                coord.y++;\n\
                tmpSum = tmpSum + data.x;\n\
                tmpSqr = tmpSqr + data.x * data.x;\n\
            }\n\
            sqr += (tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp) * e2InScale;\n\
            sum += (tmpSum - height * input_zp) * input_scale;\n\
        }\n\
    }\n\
    lcl_sum[lidx] = sum;\n\
    lcl_sqr[lidx] = sqr;\n\
    barrier(CLK_LOCAL_MEM_FENCE);\n\
\n\
    int2 coord_out = (int2)(0, 0);\n\
    if(lidx == 0)\n\
    {\n\
        float4 one = (float4)(1, 1, 1, 1);\n\
        __local float4* tmp_sum = (__local float4*)lcl_sum;\n\
        __local float4* tmp_sqr = (__local float4*)lcl_sqr;\n\
\n\
        sum = 0; sqr = 0;\n\
        for(int i = 0; i < 4; i++)\n\
        {\n\
            sum += dot(tmp_sum[i], one);\n\
            sqr += dot(tmp_sqr[i], one);\n\
        }\n\
\n\
        float4 mean, vari;\n\
        mean.x = sum * dimRatio;\n\
        vari.x = sqr * dimRatio;\n\
        vari.x = vari.x - mean.x * mean.x;\n\
        write_imagef(output_mean, coord_out, mean);\n\
        write_imagef(output_vari, coord_out, vari);\n\
    }\n\
}\n\
"; /* end of moments_axis012_cl*/

static const char moments_axis1_cl[] = "__kernel void moments_axis1_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis, int axis_num, int input_zp, float input_scale,\n\
    int width, int height, int chn, float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, 0, gidz, 0);\n\
    uint data;\n\
    float sum = 0, sqr = 0;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    {\n\
        for(coord0.y = 0; coord0.y < height;)\n\
        {\n\
            data = read_imageui(input, coord0).x;\n\
            coord0.y++;\n\
            tmpSum += (data);\n\
            tmpSqr += (data * data);\n\
        }\n\
        sqr = convert_float(as_int(tmpSqr - 2 * input_zp * tmpSum + height * input_zp * input_zp)) * e2InScale;\n\
        sum = convert_float(as_int(tmpSum - height * input_zp)) * input_scale;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
#define MOMENTS_AXIS1_F(src0_type_name) \\\n\
__kernel void moments_axis1_##src0_type_name##to##src0_type_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_t  output_mean, \\\n\
    __write_only image2d_t  output_vari, \\\n\
    int axis, int axis_num, int input_zp, float input_scale, \\\n\
    int width, int height, int chn, float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidz = get_global_id(1); \\\n\
 \\\n\
    int4 coord0 = (int4)(gidx, 0, gidz, 0); \\\n\
    float data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    for(coord0.y = 0; coord0.y < height;) \\\n\
    { \\\n\
        data = read_imagef(input, coord0).x; \\\n\
        coord0.y++; \\\n\
        sum += (data); \\\n\
        sqr += (data * data); \\\n\
    } \\\n\
 \\\n\
    float4 mean, vari; \\\n\
    mean.x = sum * dimRatio; \\\n\
    vari.x = sqr * dimRatio; \\\n\
    vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidz); \\\n\
    write_imagef(output_mean, coord_out, mean); \\\n\
    write_imagef(output_vari, coord_out, vari); \\\n\
}\n\
MOMENTS_AXIS1_F(F32)\n\
\n\
__kernel void moments_axis1_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidz = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, 0, gidz, 0);\n\
    float data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.y = 0; coord0.y < height;)\n\
    {\n\
        data = convert_float(read_imagei(input, coord0).x);\n\
        coord0.y++;\n\
        sum = sum + data;\n\
        sqr = sqr + data * data;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio * input_scale;\n\
    vari.x = sqr * dimRatio * input_scale * input_scale;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidz);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}"; /* end of moments_axis1_cl*/

static const char moments_axis2_cl[] = "__kernel void moments_axis2_U8toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, gidy, 0, 0);\n\
    uint data;\n\
    float sum = 0, sqr = 0;\n\
    uint tmpSum = 0, tmpSqr = 0;\n\
    float e2InScale = input_scale * input_scale;\n\
\n\
    {\n\
        for(coord0.z = 0; coord0.z < chn;)\n\
        {\n\
            data = read_imageui(input, coord0).x;\n\
            coord0.z++;\n\
            tmpSum += (data);\n\
            tmpSqr += (data * data);\n\
        }\n\
        sqr = as_int(tmpSqr - 2 * input_zp * tmpSum + chn * input_zp * input_zp) * e2InScale;\n\
        sum = tmpSum * input_scale;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio - input_zp * input_scale;\n\
    vari.x = sqr * dimRatio;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}\n\
\n\
#define MOMENTS_AXIS2_F(src0_type_name) \\\n\
__kernel void moments_axis2_##src0_type_name##to##src0_type_name( \\\n\
    __read_only image2d_array_t   input, \\\n\
    __write_only image2d_t  output_mean, \\\n\
    __write_only image2d_t  output_vari, \\\n\
    int axis, \\\n\
    int axis_num, \\\n\
    int input_zp, \\\n\
    float input_scale, \\\n\
    int width, \\\n\
    int height, \\\n\
    int chn, \\\n\
    float dimRatio \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
 \\\n\
    int4 coord0 = (int4)(gidx, gidy, 0, 0); \\\n\
    float data; \\\n\
    float sum = 0, sqr = 0; \\\n\
 \\\n\
    for(coord0.z = 0; coord0.z < chn;) \\\n\
    { \\\n\
        data = read_imagef(input, coord0).x; \\\n\
        coord0.z++; \\\n\
        sum += (data); \\\n\
        sqr += (data * data); \\\n\
    } \\\n\
 \\\n\
    float4 mean, vari; \\\n\
    mean.x = sum * dimRatio; \\\n\
    vari.x = sqr * dimRatio; \\\n\
    vari.x = vari.x - mean.x * mean.x; \\\n\
 \\\n\
    int2 coord_out = (int2)(gidx, gidy); \\\n\
    write_imagef(output_mean, coord_out, mean); \\\n\
    write_imagef(output_vari, coord_out, vari); \\\n\
}\n\
MOMENTS_AXIS2_F(F32)\n\
\n\
__kernel void moments_axis2_I32toF32(\n\
    __read_only image2d_array_t   input,\n\
    __write_only image2d_t  output_mean,\n\
    __write_only image2d_t  output_vari,\n\
    int axis,\n\
    int axis_num,\n\
    int input_zp,\n\
    float input_scale,\n\
    int width,\n\
    int height,\n\
    int chn,\n\
    float dimRatio\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
\n\
    int4 coord0 = (int4)(gidx, gidy, 0, 0);\n\
    float data;\n\
    float sum = 0, sqr = 0;\n\
\n\
    for(coord0.z = 0; coord0.z < chn;)\n\
    {\n\
        data = convert_float(read_imagei(input, coord0).x);\n\
        coord0.z++;\n\
\n\
\n\
        sum = sum + data;\n\
        sqr = sqr + data * data;\n\
    }\n\
\n\
    float4 mean, vari;\n\
    mean.x = sum * dimRatio * input_scale;\n\
    vari.x = sqr * dimRatio * input_scale * input_scale;\n\
    vari.x = vari.x - mean.x * mean.x;\n\
\n\
    int2 coord_out = (int2)(gidx, gidy);\n\
    write_imagef(output_mean, coord_out, mean);\n\
    write_imagef(output_vari, coord_out, vari);\n\
}"; /* end of moments_axis2_cl*/

static const char one_hot_cl[] = "__kernel void one_hot_F32toF32\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     float           on_value,\n\
                     float           off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    float4 val = read_imagef(input, coord.xy);\n\
\n\
    do\n\
    {\n\
        float4 dst;\n\
        dst.x = convert_int(val.x) == coord.z ? on_value : off_value;\n\
\n\
        write_imagef(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_I32toI32\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     int             on_value,\n\
                     int             off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 val = read_imagei(input, coord.xy);\n\
\n\
    do\n\
    {\n\
        int4 dst;\n\
        dst.x = val.x == coord.z ? on_value : off_value;\n\
\n\
        write_imagei(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_I32toU8\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     uint            on_value,\n\
                     uint            off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 val = read_imagei(input, coord.xy);\n\
    do\n\
    {\n\
        uint4 dst;\n\
        dst.x = val.x == coord.z ? on_value : off_value;\n\
\n\
        write_imageui(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_I32toF32\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     float           on_value,\n\
                     float           off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    int4 val = read_imagei(input, coord.xy);\n\
\n\
    do\n\
    {\n\
        float4 dst;\n\
        dst.x = val.x == coord.z ? on_value : off_value;\n\
\n\
        write_imagef(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
\n\
__kernel void one_hot_U8toU8\n\
    (\n\
        __read_only  image2d_t       input,\n\
        __write_only image2d_array_t output,\n\
                     int             depth,\n\
                     uint            on_value,\n\
                     uint            off_value,\n\
                     float           inputScale,\n\
                     float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
\n\
    uint4 src = read_imageui(input, coord.xy);\n\
\n\
    int  val = convert_int(convert_float(src.x) * inputScale - inputTail);\n\
\n\
    do\n\
    {\n\
        uint4 dst;\n\
        dst.x = val == coord.z ? on_value : off_value;\n\
\n\
        write_imageui(output, coord.xzyw, dst.xxxx);\n\
\n\
        coord.z ++;\n\
    } while (coord.z < depth);\n\
}\n\
"; /* end of one_hot_cl*/

static const char poolwithargmax_cl[] = "\n\
#define POOLWITHARGMAX_PROCESS(data_type, read_fun, write_fun0, write_fun1) \\\n\
    data_type src  = 0; \\\n\
    data_type max  = 0; \\\n\
    uint4  axis = 0; \\\n\
    src.x = read_fun(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_fun(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_fun(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_fun(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    write_fun0(output,  coord_out, max); \\\n\
    write_fun1(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_F32to_F32_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_PROCESS(float4, read_imagef, write_imagef, write_imageui)\n\
}\n\
\n\
__kernel void poolwithargmax_F32to_F32_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_PROCESS(float4, read_imagef, write_imagef, write_imageui)\n\
}\n\
\n\
__kernel void poolwithargmax_I32to_I32_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_PROCESS(int4, read_imagei, write_imagei, write_imageui)\n\
}\n\
\n\
__kernel void poolwithargmax_I32to_I32_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_PROCESS(int4, read_imagei, write_imagei, write_imageui)\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_U8_PROCESS() \\\n\
    uint4 src  = 0; \\\n\
    uint4 max  = 0; \\\n\
    uint4 axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x = read_imageui(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_imageui(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_imageui(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_imageui(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    result.x = convert_float4(max).x * scale_value + tail_value; \\\n\
    max = convert_uint4(result);\\\n\
    write_imageui(output,  coord_out, max); \\\n\
    write_imageui(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis,\n\
                           float   scale_value,\n\
                           float   tail_value)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_U8_PROCESS()\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_U8_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis,\n\
                     float   scale_value,\n\
                     float   tail_value)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_U8_PROCESS()\n\
}\n\
\n\
\n\
#define POOLWITHARGMAX_U8_TO_F32_PROCESS() \\\n\
    uint4 src  = 0; \\\n\
    uint4 max  = 0; \\\n\
    uint4 axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x = read_imageui(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_imageui(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_imageui(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_imageui(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    result.x = convert_float4(max).x * scale_value + tail_value; \\\n\
    write_imagef(output,  coord_out, result); \\\n\
    write_imageui(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_U8to_F32_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis,\n\
                           float   scale_value,\n\
                           float   tail_value)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_U8_TO_F32_PROCESS()\n\
}\n\
\n\
__kernel void poolwithargmax_U8to_F32_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis,\n\
                     float   scale_value,\n\
                     float   tail_value)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_U8_TO_F32_PROCESS()\n\
}\n\
\n\
#define POOLWITHARGMAX_F32_TO_U8_PROCESS() \\\n\
    float4 src  = 0; \\\n\
    float4 max  = 0; \\\n\
    uint4 axis = 0; \\\n\
    uint4 dst  = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x = read_imagef(input, coord_in).x; \\\n\
    coord_in.x++; \\\n\
    src.y = read_imagef(input, coord_in).x; \\\n\
    coord_in.y++; \\\n\
    src.w = read_imagef(input, coord_in).x; \\\n\
    coord_in.x--; \\\n\
    src.z = read_imagef(input, coord_in).x; \\\n\
    max.x  = src.x; \\\n\
    axis.x = 0; \\\n\
    if (src.y > max.x) \\\n\
    { \\\n\
        max.x  = src.y; \\\n\
        axis.x = 1; \\\n\
    } \\\n\
    if (src.z > max.x) \\\n\
    { \\\n\
        max.x  = src.z; \\\n\
        axis.x = 2; \\\n\
    } \\\n\
    if (src.w > max.x) \\\n\
    { \\\n\
        max.x  = src.w; \\\n\
        axis.x = 3; \\\n\
    } \\\n\
    result.x = max.x * scale_value + tail_value; \\\n\
    dst = convert_uint4(result);\\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    write_imageui(outaxis, coord_out, axis);\n\
\n\
\n\
__kernel void poolwithargmax_F32to_U8_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   output,\n\
    __write_only image2d_array_t   outaxis,\n\
                           float   scale_value,\n\
                           float   tail_value)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    POOLWITHARGMAX_F32_TO_U8_PROCESS()\n\
}\n\
\n\
__kernel void poolwithargmax_F32to_U8_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   output,\n\
    __write_only image2d_t   outaxis,\n\
                     float   scale_value,\n\
                     float   tail_value)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in  =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    POOLWITHARGMAX_F32_TO_U8_PROCESS()\n\
}\n\
"; /* end of poolwithargmax_cl*/

static const char pow_cl[] = "__kernel void pow_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0, src1;\n\
    float4 dst;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4  s0 = sign(src0);\n\
    int4 t0 = convert_int4(src1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
    dst.x = (src0.x == 0 && src1.x == 0) ? 1.0f : (src0.x != 0 ? (s0.x * exp2(src1.x * log2(fabs(src0.x)))) : 0.0f);\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void pow_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 dst = (float4)(0);\n\
\n\
    float4  s0 = sign(src0);\n\
    int4 t0 = convert_int4(src1) & 1;\n\
    s0 = s0 == -1 ? convert_float4(t0) == 1.0f ? -1.0f : 1.0f : s0;\n\
\n\
    dst.x = (src0.x == 0 && src1.x == 0) ? 1.0f : (src0.x != 0 ? (s0.x * exp2(src1.x * log2(fabs(src0.x)))) : 0.0f);\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of pow_cl*/

static const char prelu_cl[] = "__kernel void prelu_FP32FP32toFP32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    float4 src0;\n\
    float4 src1;\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
\n\
    float4 maxData = src0 >= 0 ? src0 : 0;\n\
    float4 minData = src0 < 0 ? src0 : 0;\n\
    float4 dst = maxData + minData * src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_FP32FP32toFP32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    float4 src0 = read_imagef(input0, coord);\n\
    float4 src1 = read_imagef(input1, coord);\n\
\n\
    float4 maxData = src0 >= 0 ? src0 : 0;\n\
    float4 minData = src0 < 0 ? src0 : 0;\n\
    float4 dst = maxData + minData * src1;\n\
\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_U8U8toU8\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    uint4 src0;\n\
    uint4 src1;\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_U8U8toU8_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    uint4 src0 = read_imageui(input0, coord);\n\
    uint4 src1 = read_imageui(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
\n\
__kernel void prelu_I32I32toI32\n\
    (\n\
    __read_only  image2d_array_t    input0,\n\
    __read_only  image2d_array_t    input1,\n\
    __write_only image2d_array_t    output,\n\
                 float              input0Scale,\n\
                 float              input0Tail,\n\
                 float              input1Scale,\n\
                 float              input1Tail,\n\
                 float              outputScale,\n\
                 float              outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
\n\
    int4 src0;\n\
    int4 src1;\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    int4 dst = convert_int4(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void prelu_I32I32toI32_2D\n\
    (\n\
    __read_only  image2d_t    input0,\n\
    __read_only  image2d_t    input1,\n\
    __write_only image2d_t    output,\n\
                 float        input0Scale,\n\
                 float        input0Tail,\n\
                 float        input1Scale,\n\
                 float        input1Tail,\n\
                 float        outputScale,\n\
                 float        outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
\n\
    int4 src0 = read_imagei(input0, coord);\n\
    int4 src1 = read_imagei(input1, coord);\n\
\n\
    float4 data0 = convert_float4(src0) * input0Scale - input0Tail;\n\
    float4 data1 = convert_float4(src1) * input1Scale - input1Tail;\n\
\n\
    float4 maxData = data0 >= 0 ? data0 : 0;\n\
    float4 minData = data0 < 0 ? data0 : 0;\n\
    float4 data = maxData + minData * data1;\n\
\n\
    int4 dst = convert_int4(data * outputScale + outputZP);\n\
\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
"; /* end of prelu_cl*/

static const char random_multinomial_cl[] = "#pragma OPENCL EXTENSION CL_VIV_asm : enable\n\
\n\
uint4 _philox4x32bumpkey(uint4 key)\n\
{\n\
    uint4 mask = (uint4)((uint)0x9E3779B9, (uint)0xBB67AE85, 0, 0);\n\
    //key.x += ((uint)0x9E3779B9);\n\
    //key.y += ((uint)0xBB67AE85);\n\
    key += mask;\n\
    return key;\n\
}\n\
\n\
uint mullo32(uint a, uint b)\n\
{\n\
    return a * b;\n\
}\n\
\n\
uint mulhi32(uint a, uint b)\n\
{\n\
    return mul_hi(a, b);\n\
}\n\
\n\
uint4 _philox4x32round(uint4 ctr, uint4 key)\n\
{\n\
    uint PHILOX_M4x32_0 = ((uint)0xD2511F53);\n\
    uint PHILOX_M4x32_1 = ((uint)0xCD9E8D57);\n\
\n\
    uint lo0 = mullo32(PHILOX_M4x32_0, ctr.x);\n\
    uint hi0 = mulhi32(PHILOX_M4x32_0, ctr.x);\n\
    uint lo1 = mullo32(PHILOX_M4x32_1, ctr.z);\n\
    uint hi1 = mulhi32(PHILOX_M4x32_1, ctr.z);\n\
\n\
    uint4 out = (uint4)(hi1^ctr.y^key.x, lo1, hi0^ctr.w^key.y, lo0);\n\
    return out;\n\
}\n\
\n\
uint4 philox4x32_R_10(uint4 ctr, uint4 key)\n\
{\n\
    uint i;\n\
    ctr = _philox4x32round(ctr, key);\n\
    for (i = 1; i < 10; i++)\n\
    {\n\
        key = _philox4x32bumpkey(key);\n\
        ctr = _philox4x32round(ctr, key);\n\
    }\n\
    return ctr;\n\
}\n\
\n\
__kernel void random_seed(\n\
    __read_only  image2d_array_t  seeds,\n\
    __write_only image2d_array_t  output,\n\
                 int              iter,\n\
                 float            re_rand_max\n\
    )\n\
{\n\
    Tensor s_tensor = create_tensor_from_image2d_array(seeds, 4);\n\
    __global uint* seeds_ptr = (__global uint*)s_tensor.ptr;\n\
    seeds_ptr = seeds_ptr;\n\
    uint4 key = vload4(0, seeds_ptr);\n\
\n\
    uint4 ctr = (uint4)(0);\n\
    float4 result = 0;\n\
\n\
    Tensor o_tensor = create_tensor_from_image2d_array(output, 4);\n\
    __global float* output_ptr = (__global float*)o_tensor.ptr;\n\
\n\
    for(int i = 0; i < iter; i++)\n\
    {\n\
        ctr = philox4x32_R_10(ctr, key);\n\
        result = convert_float4(ctr) * re_rand_max;\n\
        vstore4(result, i, output_ptr);\n\
    }\n\
}\n\
\n\
#define logE    (1.44269502f)\n\
float eltwise_unary_exp(float x)\n\
{\n\
    x *= logE;\n\
    x = exp2(x);\n\
    return x;\n\
}\n\
// N times of 8\n\
// x dim = 1\n\
\n\
__kernel void random_multinomial_cdf_F32\n\
    (\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output\n\
    )\n\
{\n\
    int2 coord = (int2)(0, get_global_id(1));\n\
    int class_max_iter = get_image_width(input);\n\
    float4 src0, data;\n\
    float4 dst = 0;\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
\n\
    for(coord.x = 1; coord.x < class_max_iter;)\n\
    {\n\
        src0 = read_imagef(input, coord);\n\
        coord.x ++;\n\
\n\
        maxVal = maxVal > src0 ? maxVal : src0;\n\
    }\n\
\n\
    for(coord.x = 0; coord.x < class_max_iter; )\n\
    {\n\
        float4 val;\n\
        src0 = read_imagef(input, coord);\n\
\n\
        data = src0 - maxVal;\n\
        val.x = eltwise_unary_exp(data.x);\n\
        val.x += dst.x;\n\
        dst.x = val.x;\n\
        write_imagef(output, coord.xy, val);\n\
        coord.x ++;\n\
    }\n\
}\n\
\n\
uint upper_bound(float* a, int n, float x)\n\
{\n\
    uint l = 0;\n\
    uint h = n;\n\
    while (l < h) {\n\
        int mid = (l + h) >> 1;\n\
        if (x >= a[mid]) {\n\
            l = mid + 1;\n\
        } else {\n\
            h = mid;\n\
        }\n\
    }\n\
    return l;\n\
}\n\
\n\
// one thread calculate 4\n\
__kernel void random_multinomial\n\
    (\n\
    __read_only image2d_array_t randoms,\n\
    __read_only image2d_array_t cdfs,\n\
   __write_only image2d_array_t output\n\
    )\n\
{\n\
    int gidx = get_global_id(0);\n\
    int gidy = get_global_id(1);\n\
    int4 coord = (int4)(gidx, gidy, 0, 0);\n\
    int class_size = get_image_width(cdfs);\n\
\n\
    int offset = gidy * class_size;\n\
    Tensor cdf_tensor = create_tensor_from_image2d_array(cdfs, 4);\n\
    __global float* cdf_ptr = (__global uint*)cdf_tensor.ptr;\n\
    __global float* cdfPtr = cdf_ptr + offset;\n\
\n\
    int width = get_image_width(randoms);\n\
    offset = coord.x + coord.y * width;\n\
    Tensor r_tensor = create_tensor_from_image2d_array(randoms, 4);\n\
    __global float* randoms_ptr = (__global float*)r_tensor.ptr;\n\
    randoms_ptr = randoms_ptr + offset;\n\
\n\
    width = get_image_width(output);\n\
    offset = coord.x + coord.y * width;\n\
    Tensor o_tensor = create_tensor_from_image2d_array(output, 4);\n\
    __global uint* output_ptr = (__global uint*)o_tensor.ptr;\n\
    output_ptr = output_ptr + offset;\n\
\n\
    float4 ran = vload4(0, randoms_ptr);\n\
    float total = cdfPtr[class_size - 1];\n\
    float4 target = ran * total;\n\
\n\
    uint4 out_class = (uint4)(0);\n\
    out_class.x = upper_bound(cdfPtr, class_size, target.x);\n\
    out_class.y = upper_bound(cdfPtr, class_size, target.y);\n\
    out_class.z = upper_bound(cdfPtr, class_size, target.z);\n\
    out_class.w = upper_bound(cdfPtr, class_size, target.w);\n\
\n\
    vstore4(out_class, 0, output_ptr);\n\
}\n\
\n\
"; /* end of random_multinomial_cl*/

static const char reduceall_internal_axis0_cl[] = "__kernel void reduceall_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 allVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.x ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    write_imagei(output, coord.yz, allVal);\n\
}\n\
\n\
__kernel void reduceall_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 allVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.x ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, allVal);\n\
}\n\
\n\
"; /* end of reduceall_internal_axis0_cl*/

static const char reduceall_internal_axis1_cl[] = "__kernel void reduceall_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 allVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.y ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    write_imagei(output, coord.xz, allVal);\n\
}\n\
\n\
__kernel void reduceall_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 allVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.y ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    coord.y = 0;\n\
    write_imagei(output, coord, allVal);\n\
}\n\
\n\
"; /* end of reduceall_internal_axis1_cl*/

static const char reduceall_internal_axis2_cl[] = "__kernel void reduceall_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 allVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        allVal = val && allVal;\n\
        coord.z ++;\n\
    }\n\
    allVal.x = allVal.x & 1;\n\
    write_imagei(output, coord.xy, allVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reduceall_internal_axis2_cl*/

static const char reduceany_internal_axis0_cl[] = "__kernel void reduceany_axis0_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 anyVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.x ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    write_imagei(output, coord.yz, anyVal);\n\
}\n\
\n\
__kernel void reduceany_axis0_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 anyVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.x ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, anyVal);\n\
}\n\
\n\
"; /* end of reduceany_internal_axis0_cl*/

static const char reduceany_internal_axis1_cl[] = "__kernel void reduceany_axis1_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 anyVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.y ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    write_imagei(output, coord.xz, anyVal);\n\
}\n\
\n\
__kernel void reduceany_axis1_I8toI8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 anyVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.y ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    coord.y = 0;\n\
    write_imagei(output, coord, anyVal);\n\
}\n\
\n\
"; /* end of reduceany_internal_axis1_cl*/

static const char reduceany_internal_axis2_cl[] = "__kernel void reduceany_axis2_I8toI8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 anyVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        anyVal = val || anyVal;\n\
        coord.z ++;\n\
    }\n\
    anyVal.x = anyVal.x & 1;\n\
    write_imagei(output, coord.xy, anyVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reduceany_internal_axis2_cl*/

static const char reducemax_internal_axis0_cl[] = "__kernel void reducemax_axis0_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord    =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize  = get_image_width(input);\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagef(output, coord.yz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis0_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagef(output, coord.yx, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis0_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.yz, dst);\n\
}\n\
\n\
__kernel void reducemax_axis0_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    coord.x = 0;\n\
    write_imageui(output, coord.yx, dst);\n\
}\n\
\n\
__kernel void reducemax_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 maxVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 maxVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, maxVal);\n\
}\n\
\n\
"; /* end of reducemax_internal_axis0_cl*/

static const char reducemax_internal_axis1_cl[] = "__kernel void reducemax_axis1_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis1_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagef(output, coord, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis1_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xz, dst);\n\
}\n\
\n\
__kernel void reducemax_axis1_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    coord.y = 0;\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void reducemax_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 maxVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, maxVal);\n\
}\n\
\n\
__kernel void reducemax_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 maxVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, maxVal);\n\
}\n\
\n\
"; /* end of reducemax_internal_axis1_cl*/

static const char reducemax_internal_axis2_cl[] = "__kernel void reducemax_axis2_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    float4 maxVal = read_imagef(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, maxVal);\n\
}\n\
\n\
\n\
__kernel void reducemax_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
    uint4 dst;\n\
    uint4 maxVal = read_imageui(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.z ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(maxVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xy, dst);\n\
}\n\
\n\
\n\
__kernel void reducemax_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 maxVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        maxVal = val > maxVal ? val : maxVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, maxVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reducemax_internal_axis2_cl*/

static const char reducemin_internal_axis0_cl[] = "__kernel void reducemin_axis0_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord    =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize  = get_image_width(input);\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagef(output, coord.yz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis0_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagef(output, coord.yx, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis0_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.yz, dst);\n\
}\n\
\n\
__kernel void reducemin_axis0_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    coord.x = 0;\n\
    write_imageui(output, coord.yx, dst);\n\
}\n\
\n\
__kernel void reducemin_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 minVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 minVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, minVal);\n\
}\n\
\n\
"; /* end of reducemin_internal_axis0_cl*/

static const char reducemin_internal_axis1_cl[] = "__kernel void reducemin_axis1_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis1_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagef(output, coord, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis1_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xz, dst);\n\
}\n\
\n\
__kernel void reducemin_axis1_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    coord.y = 0;\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void reducemin_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, minVal);\n\
}\n\
\n\
__kernel void reducemin_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, minVal);\n\
}\n\
\n\
"; /* end of reducemin_internal_axis1_cl*/

static const char reducemin_internal_axis2_cl[] = "__kernel void reducemin_axis2_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    float4 minVal = read_imagef(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, minVal);\n\
}\n\
\n\
\n\
__kernel void reducemin_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
    uint4 dst;\n\
    uint4 minVal = read_imageui(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        uint4 val = read_imageui(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
    dst = convert_uint4(convert_float4(minVal) * inputScale + inputTail);\n\
    write_imageui(output, coord.xy, dst);\n\
}\n\
\n\
\n\
__kernel void reducemin_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 minVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        minVal = val < minVal ? val : minVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, minVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reducemin_internal_axis2_cl*/

static const char reduceprod_internal_axis0_cl[] = "__kernel void reduceprod_axis0_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord    =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize  = get_image_width(input);\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagef(output, coord.yz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis0_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagef(output, coord.yx, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis0_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputTail\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    write_imageui(output, coord.yz, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis0_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    coord.x = 0;\n\
    write_imageui(output, coord.yx, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis0_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord   =  (int4)(0, get_global_id(0), get_global_id(1), 0);\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 prodVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    write_imagei(output, coord.yz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis0_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord   =  (int2)(0, get_global_id(0));\n\
    int axisSize = get_image_width(input);\n\
\n\
    int4 prodVal  = read_imagei(input, coord);\n\
    coord.x ++;\n\
\n\
    for (; coord.x < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.x ++;\n\
    }\n\
\n\
    coord.x = 0;\n\
    write_imagei(output, coord.yx, prodVal);\n\
}\n\
\n\
"; /* end of reduceprod_internal_axis0_cl*/

static const char reduceprod_internal_axis1_cl[] = "__kernel void reduceprod_axis1_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis1_F32toF32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagef(output, coord, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis1_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    write_imageui(output, coord.xz, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis1_U8toU8_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output,\n\
                 float     inputScale,\n\
                 float     inputTail,\n\
                 float     outputScale,\n\
                 float     outputTail\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    coord.y = 0;\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void reduceprod_axis1_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), 0, get_global_id(1), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 prodVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xz, prodVal);\n\
}\n\
\n\
__kernel void reduceprod_axis1_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input,\n\
    __write_only image2d_t output\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), 0);\n\
    int axisSize = get_image_height(input);\n\
\n\
    int4 prodVal = read_imagei(input, coord);\n\
    coord.y ++;\n\
\n\
    for (; coord.y < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.y ++;\n\
    }\n\
\n\
    coord.y = 0;\n\
    write_imagei(output, coord, prodVal);\n\
}\n\
\n\
"; /* end of reduceprod_internal_axis1_cl*/

static const char reduceprod_internal_axis2_cl[] = "__kernel void reduceprod_axis2_F32toF32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    float4 prodVal = read_imagef(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = read_imagef(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagef(output, coord.xy, prodVal);\n\
}\n\
\n\
\n\
__kernel void reduceprod_axis2_U8toU8\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputTail\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
    uint4 dst;\n\
    float4 prodVal = convert_float4(read_imageui(input, coord));\n\
    prodVal = prodVal * inputScale + inputTail;\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        float4 val = convert_float4(read_imageui(input, coord));\n\
        val = val * inputScale + inputTail;\n\
        prodVal = val * prodVal;\n\
        coord.z ++;\n\
    }\n\
    dst = convert_uint4(prodVal * outputScale + outputTail);\n\
    write_imageui(output, coord.xy, dst);\n\
}\n\
\n\
\n\
__kernel void reduceprod_axis2_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input,\n\
    __write_only image2d_t       output\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int axisSize = get_image_depth(input);\n\
\n\
    int4 prodVal = read_imagei(input, coord);\n\
    coord.z ++;\n\
\n\
    for (; coord.z < axisSize;)\n\
    {\n\
        int4 val = read_imagei(input, coord);\n\
        prodVal = val * prodVal;\n\
        coord.z ++;\n\
    }\n\
\n\
    write_imagei(output, coord.xy, prodVal);\n\
}\n\
\n\
\n\
\n\
"; /* end of reduceprod_internal_axis2_cl*/

static const char relational_ops_cl[] = "\n\
#define COMPARISONS_F32(func_name, comp_op) \\\n\
__kernel void func_name##_F32F32toBOOL8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input0Scale, \\\n\
                 float           input0Tail, \\\n\
                 float           input1Scale, \\\n\
                 float           input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    float4 src0; \\\n\
    float4 src1; \\\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord); \\\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_F32(less, <)\n\
COMPARISONS_F32(great, >)\n\
COMPARISONS_F32(less_equal, <=)\n\
COMPARISONS_F32(great_equal, >=)\n\
COMPARISONS_F32(equal, ==)\n\
COMPARISONS_F32(not_equal, !=)\n\
\n\
#define COMPARISONS_F32_2D(func_name, comp_op) \\\n\
__kernel void func_name##_F32F32toBOOL8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input0Scale, \\\n\
                 float     input0Tail, \\\n\
                 float     input1Scale, \\\n\
                 float     input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    float4 src0 = read_imagef(input0, coord); \\\n\
    float4 src1 = read_imagef(input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_F32_2D(less, <)\n\
COMPARISONS_F32_2D(great, >)\n\
COMPARISONS_F32_2D(less_equal, <=)\n\
COMPARISONS_F32_2D(great_equal, >=)\n\
COMPARISONS_F32_2D(equal, ==)\n\
COMPARISONS_F32_2D(not_equal, !=)\n\
\n\
#define COMPARISONS_U32(func_name, comp_op) \\\n\
__kernel void func_name##_U32U32toBOOL8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input0Scale, \\\n\
                 float           input0Tail, \\\n\
                 float           input1Scale, \\\n\
                 float           input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    uint4 data0; \\\n\
    uint4 data1; \\\n\
    READ_IMAGEUI_2DARRAY(data0, input0, coord); \\\n\
    READ_IMAGEUI_2DARRAY(data1, input1, coord); \\\n\
 \\\n\
    float4 src0 = convert_float4(data0) * input0Scale - input0Tail; \\\n\
    float4 src1 = convert_float4(data1) * input1Scale - input1Tail; \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_U32(less, <)\n\
COMPARISONS_U32(great, >)\n\
COMPARISONS_U32(less_equal, <=)\n\
COMPARISONS_U32(great_equal, >=)\n\
COMPARISONS_U32(equal, ==)\n\
COMPARISONS_U32(not_equal, !=)\n\
\n\
#define COMPARISONS_U32_2D(func_name, comp_op) \\\n\
__kernel void func_name##_U32U32toBOOL8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input0Scale, \\\n\
                 float     input0Tail, \\\n\
                 float     input1Scale, \\\n\
                 float     input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    uint4 data0 = read_imageui(input0, coord); \\\n\
    uint4 data1 = read_imageui(input1, coord); \\\n\
 \\\n\
    float4 src0 = convert_float4(data0) * input0Scale - input0Tail; \\\n\
    float4 src1 = convert_float4(data1) * input1Scale - input1Tail; \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_U32_2D(less, <)\n\
COMPARISONS_U32_2D(great, >)\n\
COMPARISONS_U32_2D(less_equal, <=)\n\
COMPARISONS_U32_2D(great_equal, >=)\n\
COMPARISONS_U32_2D(equal, ==)\n\
COMPARISONS_U32_2D(not_equal, !=)\n\
\n\
#define COMPARISONS_I32(func_name, comp_op) \\\n\
__kernel void func_name##_I32I32toBOOL8 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input0, \\\n\
    __read_only  image2d_array_t input1, \\\n\
    __write_only image2d_array_t output, \\\n\
                 float           input0Scale, \\\n\
                 float           input0Tail, \\\n\
                 float           input1Scale, \\\n\
                 float           input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
 \\\n\
    int4 src0; \\\n\
    int4 src1; \\\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord); \\\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_I32(less, <)\n\
COMPARISONS_I32(great, >)\n\
COMPARISONS_I32(less_equal, <=)\n\
COMPARISONS_I32(great_equal, >=)\n\
COMPARISONS_I32(equal, ==)\n\
COMPARISONS_I32(not_equal, !=)\n\
\n\
#define COMPARISONS_I32_2D(func_name, comp_op) \\\n\
__kernel void func_name##_I32I32toBOOL8_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input0, \\\n\
    __read_only  image2d_t input1, \\\n\
    __write_only image2d_t output, \\\n\
                 float     input0Scale, \\\n\
                 float     input0Tail, \\\n\
                 float     input1Scale, \\\n\
                 float     input1Tail \\\n\
    ) \\\n\
{ \\\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1)); \\\n\
 \\\n\
    int4 src0 = read_imagei(input0, coord); \\\n\
    int4 src1 = read_imagei(input1, coord); \\\n\
 \\\n\
    int4 dst = (src0)comp_op(src1); \\\n\
    dst &= 1; \\\n\
 \\\n\
    write_imagei(output, coord, dst); \\\n\
}\n\
COMPARISONS_I32_2D(less, <)\n\
COMPARISONS_I32_2D(great, >)\n\
COMPARISONS_I32_2D(less_equal, <=)\n\
COMPARISONS_I32_2D(great_equal, >=)\n\
COMPARISONS_I32_2D(equal, ==)\n\
COMPARISONS_I32_2D(not_equal, !=)\n\
\n\
"; /* end of relational_ops_cl*/

static const char relu_keras_cl[] = "\n\
__kernel void relu_keras_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_F32toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_F32toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset,\n\
                           float  inputScale,\n\
                           float  inputTail,\n\
                           float  outputScale,\n\
                           float  outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_F32toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset,\n\
                     float  inputScale,\n\
                     float  inputTail,\n\
                     float  outputScale,\n\
                     float  outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = read_imagef(input, coord);\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset,\n\
                           float  inputScale,\n\
                           float  inputTail,\n\
                           float  outputScale,\n\
                           float  outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toU8_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset,\n\
                     float  inputScale,\n\
                     float  inputTail,\n\
                     float  outputScale,\n\
                     float  outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 result = src >= max_value ? max_value : src;\n\
    result = result < threshold ? alpha * result + offset : result;\n\
    uint4 dst = convert_uint4_rte(result * outputScale + outputZP);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  alpha,\n\
                           float  max_value,\n\
                           float  threshold,\n\
                           float  offset,\n\
                           float  inputScale,\n\
                           float  inputTail,\n\
                           float  outputScale,\n\
                           float  outputZP\n\
                           )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void relu_keras_U8toF32_2D(\n\
    __read_only  image2d_t  input,\n\
    __write_only image2d_t  output,\n\
                     float  alpha,\n\
                     float  max_value,\n\
                     float  threshold,\n\
                     float  offset,\n\
                     float  inputScale,\n\
                     float  inputTail,\n\
                     float  outputScale,\n\
                     float  outputZP\n\
                     )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    float4 src = convert_float4(read_imageui(input, coord))  * inputScale + inputTail;\n\
    float4 dst = src >= max_value ? max_value : src;\n\
    dst = dst < threshold ? alpha * dst + offset : dst;\n\
    write_imagef(output, coord, dst);\n\
}"; /* end of relu_keras_cl*/

static const char repeat_cl[] = "__kernel void repeat_I32_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.y = 0; coord.y < height;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.yw);\n\
        coord.y++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.y++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_I32_axis1(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xw);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_I32_axis2(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.z = 0; coord.z < channel;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.zw);\n\
        coord.z++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.z++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_I32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int2 coord = (int2)(0, 0);\n\
    int2 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        int4 data = read_imagei(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xy);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagei(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_axis0(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), 0, get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.y = 0; coord.y < height;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.yw);\n\
        coord.y++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.y++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_axis1(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(0, get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xw);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_axis2(\n\
    __read_only image2d_array_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_array_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), 0, 0);\n\
    int4 coord_out = coord;\n\
\n\
    for(coord.z = 0; coord.z < channel;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.zw);\n\
        coord.z++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.z++;\n\
        }\n\
    }\n\
}\n\
\n\
__kernel void repeat_F32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width, int height, int channel, int axis)\n\
{\n\
    int2 coord = (int2)(0, 0);\n\
    int2 coord_out = coord;\n\
\n\
    for(coord.x = 0; coord.x < width;)\n\
    {\n\
        float4 data = read_imagef(input0, coord);\n\
        int4 len = read_imagei(input1, coord.xy);\n\
        coord.x++;\n\
        for(int i = 0; i < len.x; i++)\n\
        {\n\
            write_imagef(output, coord_out, data);\n\
            coord_out.x++;\n\
        }\n\
    }\n\
}\n\
\n\
"; /* end of repeat_cl*/

static const char resize_1d_bilinear_cl[] = "__kernel void resize_1d_bilinear_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    int4   coord_in     = (int4)(left_x_idx, coord_out.y, coord_out.z, 0);\n\
    float4 top_l, top_r, top, bottom, dst;\n\
\n\
    top_l    = read_imagef(input, coord_in);\n\
    coord_in.x++;\n\
    top_r    = read_imagef(input, coord_in);\n\
\n\
    top_r    = top_r - top_l;\n\
    dst      = top_l + x_lerp * top_r;\n\
\n\
    write_imagef(output, coord_out, dst);\n\
\n\
}\n\
\n\
\n\
__kernel void resize_1d_bilinear_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value,\n\
                           float  in_scale,\n\
                           float  in_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    int4   coord_in     = (int4)(left_x_idx, coord_out.y, coord_out.z, 0);\n\
    float4 top_l, top_r, top;\n\
    uint4  dst;\n\
\n\
    top_l    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    top_r    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
\n\
    top_r    = top_r - top_l;\n\
    top      = top_l + x_lerp * top_r;\n\
    dst      = convert_uint4(top * out_scale + out_tail);\n\
\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_1d_bilinear_cl*/

static const char resize_1d_nearest_cl[] = "\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    float  in_x       = (convert_float(coord_out.x) + half_pixel_value) * scale_x + round_value; \\\n\
    int    in_x_idx   = convert_int(in_x); \\\n\
\n\
__kernel void resize_1d_nearest_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value,\n\
                           float  round_value)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, coord_out.y, coord_out.z, 0);\n\
    float4 dst;\n\
    dst    = read_imagef(input, coord_in);\n\
    write_imagef(output, coord_out, dst);\n\
}\n\
\n\
\n\
__kernel void resize_1d_nearest_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, coord_out.y, coord_out.z, 0);\n\
    uint4 dst;\n\
    dst    = convert_uint4(convert_float4(read_imageui(input, coord_in)) * output_scale + output_tail);\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_1d_nearest_cl*/

static const char resize_bilinear_cl[] = "__kernel void resize_bilinear_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float  y_lerp       = in_y - top_y_f;\n\
    int    top_y_idx    = convert_int(top_y_f);\n\
    int4   coord_in     = (int4)(left_x_idx, top_y_idx, coord_out.z, 0);\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom, dst;\n\
\n\
    top_l    = read_imagef(input, coord_in);\n\
    coord_in.y++;\n\
    bottom_l = read_imagef(input, coord_in);\n\
    coord_in.x++;\n\
    bottom_r = read_imagef(input, coord_in);\n\
    coord_in.y--;\n\
    top_r    = read_imagef(input, coord_in);\n\
\n\
    top_r    = top_r - top_l;\n\
    top      = top_l + x_lerp * top_r;\n\
    bottom_r = bottom_r - bottom_l;\n\
    bottom   = bottom_l + x_lerp * bottom_r;\n\
    bottom   = bottom - top;\n\
    dst      = top + y_lerp * bottom;\n\
\n\
    write_imagef(output, coord_out, dst);\n\
\n\
}\n\
\n\
\n\
__kernel void resize_bilinear_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value,\n\
                           float  in_scale,\n\
                           float  in_tail,\n\
                           float  out_scale,\n\
                           float  out_tail\n\
                           )\n\
{\n\
    int4   coord_out    =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    float  in_x         = (convert_float(coord_out.x) + half_pixel_value) * scale_x - half_pixel_value;\n\
    float  left_x_f     = floor(in_x);\n\
    float  x_lerp       = in_x - left_x_f;\n\
    int    left_x_idx   = convert_int(left_x_f);\n\
    float  in_y         = (convert_float(coord_out.y) + half_pixel_value) * scale_y - half_pixel_value;\n\
    float  top_y_f      = floor(in_y);\n\
    float  y_lerp       = in_y - top_y_f;\n\
    int    top_y_idx    = convert_int(top_y_f);\n\
    int4   coord_in     = (int4)(left_x_idx, top_y_idx, coord_out.z, 0);\n\
    float4 top_l, top_r, bottom_l, bottom_r, top, bottom;\n\
    uint4  dst;\n\
\n\
    top_l    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.y++;\n\
    bottom_l = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.x++;\n\
    bottom_r = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
    coord_in.y--;\n\
    top_r    = convert_float4(read_imageui(input, coord_in)) * in_scale + in_tail;\n\
\n\
    top_r    = top_r - top_l;\n\
    top      = top_l + x_lerp * top_r;\n\
    bottom_r = bottom_r - bottom_l;\n\
    bottom   = bottom_l + x_lerp * bottom_r;\n\
    bottom   = bottom - top;\n\
    top      = top + y_lerp * bottom;\n\
\n\
    dst      = convert_uint4(top * out_scale + out_tail);\n\
\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_bilinear_cl*/

static const char resize_nearest_cl[] = "\n\
#define NEAREST_INDEX_PROCESS() \\\n\
    int4   coord_out  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    float  in_x       = (convert_float(coord_out.x) + half_pixel_value) * scale_x + round_value; \\\n\
    int    in_x_idx   = convert_int(in_x); \\\n\
    float  in_y       = (convert_float(coord_out.y) + half_pixel_value) * scale_y + round_value; \\\n\
    int    in_y_idx   = convert_int(in_y); \\\n\
\n\
__kernel void resize_nearest_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value,\n\
                           float  round_value)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, coord_out.z, 0);\n\
    float4 dst;\n\
    dst    = read_imagef(input, coord_in);\n\
    write_imagef(output, coord_out, dst);\n\
}\n\
\n\
\n\
__kernel void resize_nearest_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                           float  scale_x,\n\
                           float  scale_y,\n\
                           float  half_pixel_value,\n\
                           float  round_value,\n\
                           float  output_scale,\n\
                           float  output_tail)\n\
{\n\
    NEAREST_INDEX_PROCESS()\n\
    int4 coord_in = (int4)(in_x_idx, in_y_idx, coord_out.z, 0);\n\
    uint4 dst;\n\
    dst    = convert_uint4(convert_float4(read_imageui(input, coord_in)) * output_scale + output_tail);\n\
    write_imageui(output, coord_out, dst);\n\
}\n\
"; /* end of resize_nearest_cl*/

static const char roi_align_cl[] = "inline float roi_align_1x1\n\
(\n\
    __read_only  image2d_array_t  input,\n\
                           float2 region_start,\n\
                           float2 region_end,\n\
                           float2 bin_size,\n\
                           int2   grid_size,\n\
                           float2 rcp_of_grid_size,\n\
                           int    pz\n\
)\n\
{\n\
    float sum = 0;\n\
\n\
    for(int iy = 0; iy < grid_size.y; ++iy)\n\
    {\n\
        for(int ix = 0; ix < grid_size.x; ++ix)\n\
        {\n\
            float2 ixy = (float2)(ix + 0.5f, iy + 0.5f);\n\
            float2 pos = region_start + ixy * bin_size * rcp_of_grid_size;\n\
\n\
            int2 xy_low  = convert_int2(pos);\n\
            int2 xy_high = xy_low + 1;\n\
\n\
            float ly = pos.y - xy_low.y;\n\
            float lx = pos.x - xy_low.x;\n\
            float hy = 1.0f - ly;\n\
            float hx = 1.0f - lx;\n\
\n\
            float w1 = hy * hx;\n\
            float w2 = hy * lx;\n\
            float w3 = ly * hx;\n\
            float w4 = ly * lx;\n\
\n\
            float data1 = read_imagef(input, (int4)(xy_low.x, xy_low.y, pz, 0)).x;\n\
            float data2 = read_imagef(input, (int4)(xy_high.x, xy_low.y, pz, 0)).x;\n\
            float data3 = read_imagef(input, (int4)(xy_low.x, xy_high.y, pz, 0)).x;\n\
            float data4 = read_imagef(input, (int4)(xy_high.x, xy_high.y, pz, 0)).x;\n\
\n\
            sum = sum + w1 * data1 + w2 * data2 + w3 * data3 + w4 * data4;\n\
        }\n\
    }\n\
\n\
    return (float)(sum * rcp_of_grid_size.x * rcp_of_grid_size.y);\n\
}\n\
\n\
\n\
#define EPS_GRID 0.00001f\n\
__kernel void roi_align_F32toF32\n\
(\n\
    __read_only  image2d_array_t  input,\n\
    __read_only  image2d_t        rois,\n\
    __read_only  image2d_t        n_rois,\n\
    __write_only image2d_array_t  output,\n\
                           float  spatial_x_scale,\n\
                           float  spatial_y_scale,\n\
                           float  in_width,\n\
                           float  in_height,\n\
                           float  rcp_of_out_width,\n\
                           float  rcp_of_out_height,\n\
                           float  sampling_x_ratio,\n\
                           float  sampling_y_ratio,\n\
                           int    depth\n\
)\n\
{\n\
    int px = get_global_id(0);\n\
    int py = get_global_id(1);\n\
    int pw = get_global_id(2);\n\
\n\
    int roi_batch = read_imagei(n_rois, (int2)(pw, 0)).x;\n\
    float4 roi_x = read_imagef(rois, (int2)(0, pw));\n\
    float4 roi_y = read_imagef(rois, (int2)(1, pw));\n\
    float4 roi_z = read_imagef(rois, (int2)(2, pw));\n\
    float4 roi_w = read_imagef(rois, (int2)(3, pw));\n\
    float4 roi = (float4)(roi_x.x, roi_y.x, roi_z.x, roi_w.x);\n\
\n\
    float4 roi_anchor = roi * (float4)(spatial_x_scale, spatial_y_scale, spatial_x_scale, spatial_y_scale);\n\
    float2 roi_dims = fmax(roi_anchor.zw - roi_anchor.xy, 1.0f);\n\
\n\
    float2 spatial_indx     = (float2)(px, py);\n\
    float2 pooled_dims      = (float2)(rcp_of_out_width, rcp_of_out_height);\n\
    float2 max_spatial_dims = (float2)(in_width, in_height);\n\
\n\
    float2 bin_size     = roi_dims * pooled_dims;\n\
    float2 region_start = spatial_indx * bin_size + roi_anchor.xy;\n\
    float2 region_end   = region_start + bin_size;\n\
\n\
    float2 roi_bin_grid = (float2)(sampling_x_ratio, sampling_y_ratio);\n\
\n\
    roi_bin_grid = roi_bin_grid == 0 ? ceil(bin_size - EPS_GRID) : roi_bin_grid;\n\
\n\
    int kz = roi_batch * depth;\n\
    float2 rcp_of_grid_size = 1.0f / roi_bin_grid;\n\
    int2 grid_size_xy = convert_int2(roi_bin_grid);\n\
    float4 interp;\n\
    int kz1 = pw * depth;\n\
    for (int pz = 0; pz < depth; pz ++, kz ++, kz1 ++)\n\
    {\n\
        interp.x = roi_align_1x1( input,\n\
                       region_start,\n\
                       region_end,\n\
                       bin_size,\n\
                       grid_size_xy,\n\
                       rcp_of_grid_size,\n\
                       kz);\n\
\n\
        write_imagef(output, (int4)(px, py, kz1, 0), interp);\n\
    }\n\
}"; /* end of roi_align_cl*/

static const char scatter_nd_cl[] = "__kernel void scatter_nd_U32toU32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    uint4 sum = (uint4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        if(gidy == indice.x)\n\
        {\n\
            uint4 data = read_imageui(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imageui(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_U32toU32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    uint4 sum = (uint4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int idx = indice0.x * width + indice1.x;\n\
        if(gidy == idx)\n\
        {\n\
            uint4 data = read_imageui(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imageui(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_U32toU32_3D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    uint4 sum = (uint4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int4 indice2 = read_imagei(input0, (int2)(2, i));\n\
        int idx = indice0.x * area + indice1.x * width + indice2.x;\n\
        if(gidy == idx)\n\
        {\n\
            uint4 data = read_imageui(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imageui(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_I32toI32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 sum = (int4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        if(gidy == indice.x)\n\
        {\n\
            int4 data = read_imagei(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagei(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_I32toI32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 sum = (int4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int idx = indice0.x * width + indice1.x;\n\
        if(gidy == idx)\n\
        {\n\
            int4 data = read_imagei(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagei(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_I32toI32_3D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    int4 sum = (int4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int4 indice2 = read_imagei(input0, (int2)(2, i));\n\
        int idx = indice0.x * area + indice1.x * width + indice2.x;\n\
        if(gidy == idx)\n\
        {\n\
            int4 data = read_imagei(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagei(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_F32toF32_1D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice = read_imagei(input0, (int2)(0, i));\n\
        if(gidy == indice.x)\n\
        {\n\
            float4 data = read_imagef(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagef(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_F32toF32_2D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int idx = indice0.x * width + indice1.x;\n\
        if(gidy == idx)\n\
        {\n\
            float4 data = read_imagef(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagef(output, (int2)(gidx, gidy), sum);\n\
}\n\
\n\
__kernel void scatter_nd_F32toF32_3D(\n\
    __read_only image2d_t   input0,\n\
    __read_only image2d_t   input1,\n\
    __write_only image2d_t  output,\n\
    int width,\n\
    int area,\n\
    int index_num\n\
    )\n\
{\n\
    int gidx = get_global_id(0);  // block_size\n\
    int gidy = get_global_id(1);  // indices_num\n\
\n\
    float4 sum = (float4)(0, 0, 0, 0);\n\
    for(int i = 0; i < index_num; i++)\n\
    {\n\
        int4 indice0 = read_imagei(input0, (int2)(0, i));\n\
        int4 indice1 = read_imagei(input0, (int2)(1, i));\n\
        int4 indice2 = read_imagei(input0, (int2)(2, i));\n\
        int idx = indice0.x * area + indice1.x * width + indice2.x;\n\
        if(gidy == idx)\n\
        {\n\
            float4 data = read_imagef(input1, (int2)(gidx, i));\n\
            sum += data;\n\
        }\n\
    }\n\
    write_imagef(output, (int2)(gidx, gidy), sum);\n\
}"; /* end of scatter_nd_cl*/

static const char scatter_nd_update_cl[] = "\n\
#define SCATTER_ND_UPDATE(src0_type, data_type, read_func, write_func) \\\n\
__kernel void scatter_nd_update_##src0_type##src0_type##to##src0_type( \\\n\
    __read_only image2d_t   input0, \\\n\
    __read_only image2d_t   input1, \\\n\
    __read_only image2d_t   input2, \\\n\
    __write_only image2d_t  output, \\\n\
    int offsetX, \\\n\
    int offsetY, \\\n\
    int offsetZ, \\\n\
    int offsetW, \\\n\
    int offset_idx, \\\n\
    int coord_dim, \\\n\
    int index_num \\\n\
    ) \\\n\
{ \\\n\
    int gidx = get_global_id(0); \\\n\
    int gidy = get_global_id(1); \\\n\
    int cnt = 0; \\\n\
 \\\n\
    data_type sum = (data_type)(0, 0, 0, 0); \\\n\
    Image img1 = create_image_from_image2d(input1, 4); \\\n\
    __global int* index_ptr = (__global int*)img1.ptr; \\\n\
    for(int i = 0; i < index_num; i++) \\\n\
    { \\\n\
        int4 indice = vload4(0, index_ptr + offset_idx); \\\n\
        index_ptr += coord_dim; \\\n\
        int idx = indice.x * offsetX + indice.y * offsetY + indice.z * offsetZ + indice.w * offsetW; \\\n\
        if(gidy == idx) \\\n\
        { \\\n\
            data_type data = read_func(input2, (int2)(gidx, i)); \\\n\
            cnt++; \\\n\
            sum += data; \\\n\
        } \\\n\
    } \\\n\
    int2 coord = (int2)(gidx, gidy); \\\n\
    if(cnt == 0) \\\n\
    { \\\n\
        sum = read_func(input0, coord); \\\n\
    } \\\n\
    write_func(output, coord, sum); \\\n\
}\n\
SCATTER_ND_UPDATE(U32,  uint4,  read_imageui, write_imageui)\n\
SCATTER_ND_UPDATE(I32,  int4,   read_imagei,  write_imagei)\n\
SCATTER_ND_UPDATE(F32,  float4, read_imagef,  write_imagef)\n\
"; /* end of scatter_nd_update_cl*/

static const char select_cl[] = "__kernel void select_I8_U8_U8toU8(\n\
    __read_only  image2d_array_t  condition,\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4  value;\n\
    uint4 src0, src1, src, dst;\n\
    float inputScale, inputTail;\n\
    READ_IMAGEI_2DARRAY(value, condition, coord);\n\
    READ_IMAGEUI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEUI_2DARRAY(src1, input1, coord);\n\
    src   = (value != 0 ? src0 : src1);\n\
    inputScale = (value.x != 0 ? input0Scale : input1Scale);\n\
    inputTail  = (value.x != 0 ? input0Tail  : input1Tail);\n\
    dst = convert_uint4(convert_float4(src) * inputScale + inputTail);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_U8_U8toU8_2D(\n\
    __read_only  image2d_t        condition,\n\
    __read_only  image2d_t        input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_t        output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4  value = read_imagei(condition, coord);\n\
    uint4 src0  = read_imageui(input0, coord);\n\
    uint4 src1  = read_imageui(input1, coord);\n\
    uint4 src   = (value != 0 ? src0 : src1);\n\
    float inputScale = (value.x != 0 ? input0Scale : input1Scale);\n\
    float inputTail  = (value.x != 0 ? input0Tail  : input1Tail);\n\
    uint4 dst = convert_uint4(convert_float4(src) * inputScale + inputTail);\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_I32_I32toI32(\n\
    __read_only  image2d_array_t  condition,\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4  value;\n\
    int4 src0, src1, dst;\n\
    READ_IMAGEI_2DARRAY(value, condition, coord);\n\
    READ_IMAGEI_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEI_2DARRAY(src1, input1, coord);\n\
    dst   = (value != 0 ? src0 : src1);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_I32_I32toI32_2D(\n\
    __read_only  image2d_t        condition,\n\
    __read_only  image2d_t        input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_t        output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 value = read_imagei(condition, coord);\n\
    int4 src0  = read_imagei(input0, coord);\n\
    int4 src1  = read_imagei(input1, coord);\n\
    int4 dst   = (value != 0 ? src0 : src1);\n\
    write_imagei(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_F32_F32toF32(\n\
    __read_only  image2d_array_t  condition,\n\
    __read_only  image2d_array_t  input0,\n\
    __read_only  image2d_array_t  input1,\n\
    __write_only image2d_array_t  output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int4 coord  = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4  value;\n\
    float4 src0, src1, dst;\n\
    READ_IMAGEI_2DARRAY(value, condition, coord);\n\
    READ_IMAGEF_2DARRAY(src0, input0, coord);\n\
    READ_IMAGEF_2DARRAY(src1, input1, coord);\n\
    dst   = (value != 0 ? src0 : src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
\n\
__kernel void select_I8_F32_F32toF32_2D(\n\
    __read_only  image2d_t        condition,\n\
    __read_only  image2d_t        input0,\n\
    __read_only  image2d_t        input1,\n\
    __write_only image2d_t        output,\n\
                 float            input0Scale,\n\
                 float            input0Tail,\n\
                 float            input1Scale,\n\
                 float            input1Tail)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int4 value = read_imagei(condition, coord);\n\
    float4 src0  = read_imagef(input0, coord);\n\
    float4 src1  = read_imagef(input1, coord);\n\
    float4 dst   = (value != 0 ? src0 : src1);\n\
    write_imagef(output, coord, dst);\n\
}\n\
"; /* end of select_cl*/

static const char sequence_mask_cl[] = "\n\
__kernel void sequence_mask_I32toU8(\n\
    image2d_t input, image2d_array_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    int4 index = read_imagei(input, coord.yz);\n\
    uint4 data;\n\
    data.x = gidx < index.x ? convert_uint_rte(outputVal1) : (uint)(output_ZP);\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toU8_2D(\n\
    image2d_t input, image2d_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int4 index = read_imagei(input, coord.yy);\n\
    uint4 data;\n\
    data.x = gidx < index.x ? convert_uint_rte(outputVal1) : (uint)(output_ZP);\n\
    write_imageui(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toI32(\n\
    image2d_t input, image2d_array_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    int4 index = read_imagei(input, coord.yz);\n\
    int4 data;\n\
    data = gidx < index.x ? (int4)(1) : (int4)(0);\n\
    write_imagei(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toI32_2D(\n\
    image2d_t input, image2d_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int4 index = read_imagei(input, coord.yy);\n\
    int4 data;\n\
    data = gidx < index.x ? (int4)(1) : (int4)(0);\n\
    write_imagei(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toF32(\n\
    image2d_t input, image2d_array_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int4 coord = (int4)(gidx, get_global_id(1), get_global_id(2), 0);\n\
    int4 index = read_imagei(input, coord.yz);\n\
    float4 data;\n\
    data = gidx < index.x ? (float4)(1.0f) : (float4)(0.0f);\n\
    write_imagef(output, coord, data);\n\
}\n\
\n\
__kernel void sequence_mask_I32toF32_2D(\n\
    image2d_t input, image2d_t output, int maxLen,\n\
    float input_scale, float input_zpScale, float outputVal1, int output_ZP)\n\
{\n\
    int gidx = get_global_id(0);\n\
    int2 coord = (int2)(gidx, get_global_id(1));\n\
    int4 index = read_imagei(input, coord.yy);\n\
    float4 data;\n\
    data = gidx < index.x ? (float4)(1.0f) : (float4)(0.0f);\n\
    write_imagef(output, coord, data);\n\
}"; /* end of sequence_mask_cl*/

static const char signal_frame_cl[] = "\n\
#define SIGNAL_FRAME_SH_IMPL(type, data_type, read_imagefunc, write_imagefunc) \\\n\
__kernel void signal_frame_##type##to##type \\\n\
    ( \\\n\
    __read_only  image2d_t       input, \\\n\
    __write_only image2d_array_t output, \\\n\
                 int             frame_step \\\n\
    ) \\\n\
{ \\\n\
    int inner = get_global_id(0); \\\n\
    int length_k = get_global_id(1); \\\n\
    int frames_id = get_global_id(2); \\\n\
 \\\n\
    int4 coord = (int4)(inner, length_k, frames_id, frames_id); \\\n\
    int2 coord_in = (int2)(inner, frames_id * frame_step + length_k); \\\n\
 \\\n\
    data_type src = read_imagefunc(input, coord_in); \\\n\
    write_imagefunc(output, coord, src); \\\n\
}\n\
SIGNAL_FRAME_SH_IMPL(F32, float4, read_imagef,  write_imagef)\n\
SIGNAL_FRAME_SH_IMPL(U8,  uint4,  read_imageui, write_imageui)\n\
"; /* end of signal_frame_cl*/

static const char slice_cl[] = "__kernel void slice_F32_I32toF32\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_t       input1,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int4 begin = ((int4 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    float4 src = read_imagef(input0, coord_in);\n\
\n\
    write_imagef(output, coord, src);\n\
}\n\
\n\
__kernel void slice_F32_I32toF32_2D\n\
    (\n\
    __read_only  image2d_t input0,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int2 begin = ((int2 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    float4 src = read_imagef(input0, coord_in);\n\
\n\
    write_imagef(output, coord, src);\n\
}\n\
\n\
__kernel void slice_U8_I32toU8\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_t       input1,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int4 begin = ((int4 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    uint4 src = read_imageui(input0, coord_in);\n\
\n\
    float4 data = convert_float4(src) * inputScale - inputTail;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void slice_U8_I32toU8_2D\n\
    (\n\
    __read_only  image2d_t input0,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int2 begin = ((int2 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    uint4 src = read_imageui(input0, coord_in);\n\
\n\
    float4 data = convert_float4(src) * inputScale - inputTail;\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP);\n\
\n\
    write_imageui(output, coord, dst);\n\
}\n\
\n\
__kernel void slice_I32_I32toI32\n\
    (\n\
    __read_only  image2d_array_t input0,\n\
    __read_only  image2d_t       input1,\n\
    __write_only image2d_array_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    int4 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int4 begin = ((int4 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    int4 src = read_imagei(input0, coord_in);\n\
\n\
    write_imagei(output, coord, src);\n\
}\n\
\n\
__kernel void slice_I32_I32toI32_2D\n\
    (\n\
    __read_only  image2d_t input0,\n\
    __read_only  image2d_t input1,\n\
    __write_only image2d_t output,\n\
                 float           inputScale,\n\
                 float           inputTail,\n\
                 float           outputScale,\n\
                 float           outputZP\n\
    )\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    int2 coord_in;\n\
    Image begin_img = create_image_from_image2d(input1, 4);\n\
    uchar* begin_ptr = begin_img.ptr;\n\
    int2 begin = ((int2 *)begin_ptr)[0];\n\
\n\
    coord_in = coord + begin;\n\
    int4 src = read_imagei(input0, coord_in);\n\
\n\
    write_imagei(output, coord, src);\n\
}\n\
\n\
"; /* end of slice_cl*/

static const char space2depth_internal_cl[] = "\n\
__kernel void space2depth_internal_F32toF32 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    float4 data = {0.0};\n\
    data = read_imagef(input, coord);\n\
\n\
    ushort blockSize_x = convert_ushort(block_size_x);\n\
    ushort blockSize_y = convert_ushort(block_size_y);\n\
    int4 coord_out = (int4)(convert_ushort(x)/blockSize_x, convert_ushort(y)/blockSize_y, 0, 0);\n\
    coord_out.z = ((x - coord_out.x * block_size_x) + (y - coord_out.y * block_size_y) * block_size_x) * inDepth\n\
                     + z;\n\
    write_imagef(output, coord_out, data);\n\
}\n\
\n\
__kernel void space2depth_internal_F32toF32_X2Y1 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    float4 data = {0.0};\n\
    data = read_imagef(input, coord);\n\
\n\
    int4 coord_out = (int4)(x >> 1, y, 0, 0);\n\
    coord_out.z = (x & 1) * inDepth + z;\n\
    write_imagef(output, coord_out, data);\n\
}\n\
\n\
__kernel void space2depth_internal_U8toU8 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    uint4 data = {0};\n\
    data = read_imageui(input, coord);\n\
\n\
    ushort blockSize_x = convert_ushort(block_size_x);\n\
    ushort blockSize_y = convert_ushort(block_size_y);\n\
    int4 coord_out = (int4)(convert_ushort(x)/blockSize_x, convert_ushort(y)/blockSize_y, 0, 0);\n\
    coord_out.z = ((x - coord_out.x * block_size_x) + (y - coord_out.y * block_size_y) * block_size_x) * inDepth\n\
                    + z;\n\
\n\
    data.x = convert_uint(data.x * scaleInOut + zpInOut);\n\
    write_imageui(output, coord_out, data);\n\
}\n\
\n\
__kernel void space2depth_internal_U8toU8_X2Y1 (\n\
        image2d_array_t    input,\n\
        image2d_array_t    output,\n\
        int block_size_x, int block_size_y,\n\
        float  scaleInOut, float zpInOut)\n\
{\n\
    int x = get_global_id(0);\n\
    int y = get_global_id(1);\n\
    int z = get_global_id(2);\n\
    int inDepth = get_image_array_size(input);\n\
\n\
    int4 coord = (int4)(x, y, z, 0);\n\
    uint4 data = {0};\n\
    data = read_imageui(input, coord);\n\
\n\
    int4 coord_out = (int4)(x >> 1, y, 0, 0);\n\
    coord_out.z = (x & 1) * inDepth + z;\n\
\n\
    data.x = convert_uint(data.x * scaleInOut + zpInOut);\n\
    write_imageui(output, coord_out, data);\n\
}\n\
"; /* end of space2depth_internal_cl*/

static const char swish_cl[] = "float sigmoid_(float x, float logE)\n\
{\n\
    x *= -logE;\n\
    x = 1 + exp2(x);\n\
    return 1 / x;\n\
}\n\
\n\
#define SWISH_F32_F32_PROCESS() \\\n\
    float4 src, tmp, dst; \\\n\
    src   = read_imagef(input, coord); \\\n\
    tmp.x = sigmoid_(src.x * beta, logE); \\\n\
    dst.x = src.x * tmp.x; \\\n\
    write_imagef(output, coord, dst);\n\
\n\
__kernel void swish_F32toF32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_F32_F32_PROCESS()\n\
}\n\
\n\
__kernel void swish_F32toF32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_F32_F32_PROCESS()\n\
}\n\
\n\
\n\
#define SWISH_U8_U8_PROCESS() \\\n\
    float4 src, tmp, data; \\\n\
    uint4 src0 = read_imageui(input, coord); \\\n\
    src   = convert_float4(src0) * inputScale - inputTail; \\\n\
    tmp.x = sigmoid_(src.x * beta, logE); \\\n\
    data.x = src.x * tmp.x; \\\n\
    uint4 dst = convert_uint4(data * outputScale + outputZP); \\\n\
    write_imageui(output, coord, dst);\n\
\n\
__kernel void swish_U8toU8(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_U8_U8_PROCESS()\n\
}\n\
\n\
__kernel void swish_U8toU8_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_U8_U8_PROCESS()\n\
}\n\
\n\
\n\
#define SWISH_I32_I32_PROCESS() \\\n\
    float4 src, tmp, data; \\\n\
    int4 src0 = read_imagei(input, coord); \\\n\
    src    = convert_float4(src0); \\\n\
    tmp.x  = sigmoid_(src.x * beta, logE); \\\n\
    data.x = src.x * tmp.x; \\\n\
    int4 dst = convert_int4(data); \\\n\
    write_imagei(output, coord, dst);\n\
\n\
__kernel void swish_I32toI32(\n\
    __read_only  image2d_array_t  input,\n\
    __write_only image2d_array_t  output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int4 coord =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    SWISH_I32_I32_PROCESS()\n\
}\n\
\n\
__kernel void swish_I32toI32_2D(\n\
    __read_only  image2d_t        input,\n\
    __write_only image2d_t        output,\n\
                 float            inputScale,\n\
                 float            inputTail,\n\
                 float            outputScale,\n\
                 float            outputZP,\n\
                 float            beta,\n\
                 float            logE)\n\
{\n\
    int2 coord =  (int2)(get_global_id(0), get_global_id(1));\n\
    SWISH_I32_I32_PROCESS()\n\
}\n\
"; /* end of swish_cl*/

static const char tile_cl[] = "\n\
#define TILE_3D(name0, name1, data_type, read_image_func, write_image_func) \\\n\
__kernel void tile_##name0##to##name1 \\\n\
    ( \\\n\
    __read_only  image2d_array_t input, \\\n\
    __write_only image2d_array_t output, \\\n\
                             int batchIn, \\\n\
                             int depthIn, \\\n\
                             int depthOut, \\\n\
                             int multiples_0, \\\n\
                             int multiples_1, \\\n\
                             int multiples_2, \\\n\
                             int multiples_3 \\\n\
    ) \\\n\
{ \\\n\
    int4 coord = (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0); \\\n\
    int4 coord_out; \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
 \\\n\
    data_type src; \\\n\
    read_image_func(src, input, coord); \\\n\
 \\\n\
    int batch_id = (short)coord.z / (short)depthIn; \\\n\
    coord.z = (short)coord.z % (short)depthIn; \\\n\
    coord_out = coord; \\\n\
 \\\n\
    for (int w = 0; w < multiples_3; w++) \\\n\
    { \\\n\
        int batch = batchIn * w + batch_id; \\\n\
 \\\n\
        for(int z = 0; z < multiples_2; z++) \\\n\
        { \\\n\
            coord_out.z = coord.z + z * depthIn + batch * depthOut; \\\n\
 \\\n\
            for (int y = 0; y < multiples_1; y++) \\\n\
            { \\\n\
                coord_out.y = coord.y + y * height; \\\n\
 \\\n\
                for (int x = 0; x < multiples_0; x++) \\\n\
                { \\\n\
                    coord_out.x = coord.x + x * width; \\\n\
                    write_image_func(output, coord_out.xyzw, src); \\\n\
                } \\\n\
            } \\\n\
        } \\\n\
    } \\\n\
}\n\
TILE_3D(I32, I32, int4,   READ_IMAGEI_2DARRAY,  write_imagei)\n\
TILE_3D(U32, U32, uint4,  READ_IMAGEUI_2DARRAY, write_imageui)\n\
TILE_3D(F32, F32, float4, READ_IMAGEF_2DARRAY,  write_imagef)\n\
\n\
#define TILE_2D(name0, name1, data_type, read_image_func, write_image_func) \\\n\
__kernel void tile_##name0##to##name1##_2D \\\n\
    ( \\\n\
    __read_only  image2d_t input, \\\n\
    __write_only image2d_t output, \\\n\
                       int batchIn, \\\n\
                       int depthIn, \\\n\
                       int depthOut, \\\n\
                       int multiples_0, \\\n\
                       int multiples_1, \\\n\
                       int multiples_2, \\\n\
                       int multiples_3 \\\n\
    ) \\\n\
{ \\\n\
    int2 coord = (int2)(get_global_id(0), get_global_id(1)); \\\n\
    int width = get_image_width(input); \\\n\
    int height = get_image_height(input); \\\n\
    int output_width = get_image_width(output); \\\n\
    int output_height = get_image_height(output); \\\n\
 \\\n\
    data_type src = read_image_func(input, coord); \\\n\
 \\\n\
    do \\\n\
    { \\\n\
        do \\\n\
        { \\\n\
            write_image_func(output, coord, src); \\\n\
            coord.x += width; \\\n\
        } while (coord.x < output_width); \\\n\
        coord.x = get_global_id(0); \\\n\
        coord.y += height; \\\n\
    } while (coord.y < output_height); \\\n\
}\n\
TILE_2D(I32, I32, int4,   read_imagei,  write_imagei)\n\
TILE_2D(U32, U32, uint4,  read_imageui, write_imageui)\n\
TILE_2D(F32, F32, float4, read_imagef,  write_imagef)\n\
\n\
\n\
\n\
"; /* end of tile_cl*/

static const char upsample_cl[] = "\n\
#define UPSAMPLE_PROCESS(data_type, read_fun, write_fun) \\\n\
    data_type src  = 0; \\\n\
    data_type dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    src.x  = read_fun(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    dst.x = axis.x == 0 ? src.x : 0; \\\n\
    write_fun(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? src.x : 0; \\\n\
    coord_out.x++; \\\n\
    write_fun(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? src.x : 0; \\\n\
    coord_out.y++; \\\n\
    write_fun(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? src.x : 0; \\\n\
    coord_out.x--; \\\n\
    write_fun(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_F32_U8to_F32(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_PROCESS(float4, read_imagef, write_imagef)\n\
}\n\
\n\
__kernel void upsample_F32_U8to_F32_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_PROCESS(float4, read_imagef, write_imagef)\n\
}\n\
\n\
__kernel void upsample_I32_U8to_I32(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_PROCESS(int4, read_imagei, write_imagei)\n\
}\n\
\n\
__kernel void upsample_I32_U8to_I32_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_PROCESS(int4, read_imagei, write_imagei)\n\
}\n\
\n\
\n\
#define UPSAMPLE_U8_PROCESS() \\\n\
    uint4  src  = 0; \\\n\
    uint4  dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    uint   output_zp = (uint)zp_out; \\\n\
    src.x  = read_imageui(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    result.x = convert_float4(src).x * scale_value + tail_value; \\\n\
    src = convert_uint4(result);\\\n\
    dst.x = axis.x == 0 ? src.x : output_zp; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? src.x : output_zp; \\\n\
    coord_out.x++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? src.x : output_zp; \\\n\
    coord_out.y++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? src.x : output_zp; \\\n\
    coord_out.x--; \\\n\
    write_imageui(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_U8_U8to_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output,\n\
                           float   scale_value,\n\
                           float   tail_value,\n\
                             int   zp_out)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_U8_PROCESS()\n\
}\n\
\n\
__kernel void upsample_U8_U8to_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output,\n\
                     float   scale_value,\n\
                     float   tail_value,\n\
                       int   zp_out)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_U8_PROCESS()\n\
}\n\
\n\
#define UPSAMPLE_U8_TO_F32PROCESS() \\\n\
    uint4  src  = 0; \\\n\
    float4  dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    src.x  = read_imageui(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    result.x = convert_float4(src).x * scale_value + tail_value; \\\n\
    dst.x = axis.x == 0 ? result.x : 0.0f; \\\n\
    write_imagef(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? result.x : 0.0f; \\\n\
    coord_out.x++; \\\n\
    write_imagef(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? result.x : 0.0f; \\\n\
    coord_out.y++; \\\n\
    write_imagef(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? result.x : 0.0f; \\\n\
    coord_out.x--; \\\n\
    write_imagef(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_U8_U8to_F32(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output,\n\
                           float   scale_value,\n\
                           float   tail_value,\n\
                             int   zp_out)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_U8_TO_F32PROCESS()\n\
}\n\
\n\
__kernel void upsample_U8_U8to_F32_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output,\n\
                     float   scale_value,\n\
                     float   tail_value,\n\
                       int   zp_out)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_U8_TO_F32PROCESS()\n\
}\n\
\n\
\n\
#define UPSAMPLE_F32_TO_U8_PROCESS() \\\n\
    uint4  src  = 0; \\\n\
    uint4  dst  = 0; \\\n\
    uint4  axis = 0; \\\n\
    float4 result = 0.0f; \\\n\
    uint   output_zp = (uint)zp_out; \\\n\
    result.x  = read_imagef(input,  coord_in).x; \\\n\
    axis.x = read_imageui(inaxis, coord_in).x; \\\n\
    result.x = result.x * scale_value + tail_value; \\\n\
    src = convert_uint4(result);\\\n\
    dst.x = axis.x == 0 ? src.x : output_zp; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 1 ? src.x : output_zp; \\\n\
    coord_out.x++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 3 ? src.x : output_zp; \\\n\
    coord_out.y++; \\\n\
    write_imageui(output,  coord_out, dst); \\\n\
    dst.x = axis.x == 2 ? src.x : output_zp; \\\n\
    coord_out.x--; \\\n\
    write_imageui(output,  coord_out, dst);\n\
\n\
\n\
__kernel void upsample_F32_U8to_U8(\n\
    __read_only  image2d_array_t   input,\n\
    __write_only image2d_array_t   inaxis,\n\
    __write_only image2d_array_t   output,\n\
                           float   scale_value,\n\
                           float   tail_value,\n\
                             int   zp_out)\n\
{\n\
    int4 coord_out =  (int4)(get_global_id(0) << 1, get_global_id(1) << 1, get_global_id(2), 0);\n\
    int4 coord_in  =  (int4)(get_global_id(0), get_global_id(1), get_global_id(2), 0);\n\
    UPSAMPLE_F32_TO_U8_PROCESS()\n\
}\n\
\n\
__kernel void upsample_F32_U8to_U8_2D(\n\
    __read_only  image2d_t   input,\n\
    __write_only image2d_t   inaxis,\n\
    __write_only image2d_t   output,\n\
                     float   scale_value,\n\
                     float   tail_value,\n\
                       int   zp_out)\n\
{\n\
    int2 coord_out =  (int2)(get_global_id(0) << 1, get_global_id(1) << 1);\n\
    int2 coord_in  =  (int2)(get_global_id(0), get_global_id(1));\n\
    UPSAMPLE_F32_TO_U8_PROCESS()\n\
}\n\
"; /* end of upsample_cl*/


typedef struct {
    char const* name;
    char const* data;
} source_map_t;

static const source_map_t evis_resource[] =
{
    {"a_times_b_plus_c_vx", a_times_b_plus_c_vx},
    {"add_mean_std_norm_vx", add_mean_std_norm_vx},
    {"argmax_axis0_vx", argmax_axis0_vx},
    {"argmax_axis1_vx", argmax_axis1_vx},
    {"argmax_axis2_vx", argmax_axis2_vx},
    {"argmin_axis0_vx", argmin_axis0_vx},
    {"argmin_axis1_vx", argmin_axis1_vx},
    {"argmin_axis2_vx", argmin_axis2_vx},
    {"batchnorm_single_vx", batchnorm_single_vx},
    {"batchnorm_single_f32_vx", batchnorm_single_f32_vx},
    {"cast_vx", cast_vx},
    {"clip_F16_vx", clip_F16_vx},
    {"clip_I16_vx", clip_I16_vx},
    {"clip_I8_vx", clip_I8_vx},
    {"clip_U8_vx", clip_U8_vx},
    {"conv1d_ovxlib_vx", conv1d_ovxlib_vx},
    {"conv1d_ovxlib_k1024_vx", conv1d_ovxlib_k1024_vx},
    {"depth2space_crd_vx", depth2space_crd_vx},
    {"depthwise_conv1d_src0_vx", depthwise_conv1d_src0_vx},
    {"depthwise_conv1d_src1_vx", depthwise_conv1d_src1_vx},
    {"depthwise_conv1d_src2_vx", depthwise_conv1d_src2_vx},
    {"depthwise_conv1d_src3_vx", depthwise_conv1d_src3_vx},
    {"detect_post_box_vx", detect_post_box_vx},
    {"eltwise_unary_2d_vx", eltwise_unary_2d_vx},
    {"eltwise_unary_3d_vx", eltwise_unary_3d_vx},
    {"erf_vx", erf_vx},
    {"extra_ending_vx", extra_ending_vx},
    {"floordiv_vx", floordiv_vx},
    {"gather_vx", gather_vx},
    {"gather_array_vx", gather_array_vx},
    {"gather_mix_vx", gather_mix_vx},
    {"gather_nd_vx", gather_nd_vx},
    {"gather_nd_2d_vx", gather_nd_2d_vx},
    {"gather_nd_2d_mix_vx", gather_nd_2d_mix_vx},
    {"gather_nd_3d_vx", gather_nd_3d_vx},
    {"gather_nd_3d_mix_vx", gather_nd_3d_mix_vx},
    {"gather_nd_mix_vx", gather_nd_mix_vx},
    {"get_matrix_vx", get_matrix_vx},
    {"group_normalization_f16_vx", group_normalization_f16_vx},
    {"group_normalization_f16_scale_vx", group_normalization_f16_scale_vx},
    {"group_normalization_i16_vx", group_normalization_i16_vx},
    {"group_normalization_i16_scale_vx", group_normalization_i16_scale_vx},
    {"group_normalization_i8_vx", group_normalization_i8_vx},
    {"group_normalization_i8_scale_vx", group_normalization_i8_scale_vx},
    {"group_normalization_u8_vx", group_normalization_u8_vx},
    {"group_normalization_u8_f16_vx", group_normalization_u8_f16_vx},
    {"grucell_activation_vx", grucell_activation_vx},
    {"grucell_activation_sma_vx", grucell_activation_sma_vx},
    {"grucell_activation_z_h_vx", grucell_activation_z_h_vx},
    {"grucell_cdnn_activation_vx", grucell_cdnn_activation_vx},
    {"grucell_cdnn_activation_u8_vx", grucell_cdnn_activation_u8_vx},
    {"grucell_h_times_activation_r_vx", grucell_h_times_activation_r_vx},
    {"grucell_reset_after_activation_vx", grucell_reset_after_activation_vx},
    {"hswish_vx", hswish_vx},
    {"instance_normalization_f16_vx", instance_normalization_f16_vx},
    {"instance_normalization_i16_vx", instance_normalization_i16_vx},
    {"instance_normalization_i8_vx", instance_normalization_i8_vx},
    {"instance_normalization_scale_f32_vx", instance_normalization_scale_f32_vx},
    {"instance_normalization_scale_f32_bf16_vx", instance_normalization_scale_f32_bf16_vx},
    {"instance_normalization_scale_f32_f16_vx", instance_normalization_scale_f32_f16_vx},
    {"instance_normalization_u8_vx", instance_normalization_u8_vx},
    {"instance_normalization_u8_f16_vx", instance_normalization_u8_f16_vx},
    {"l2normalizescale_axis0_vx", l2normalizescale_axis0_vx},
    {"l2normalizescale_axis1_vx", l2normalizescale_axis1_vx},
    {"layer_normalization_vx", layer_normalization_vx},
    {"layer_normalization_2d_vx", layer_normalization_2d_vx},
    {"layer_normalization_i16_vx", layer_normalization_i16_vx},
    {"layer_normalization_scale_f32_vx", layer_normalization_scale_f32_vx},
    {"layer_normalization_scale_f32_2d_vx", layer_normalization_scale_f32_2d_vx},
    {"layer_normalization_scale_f32_bf16_vx", layer_normalization_scale_f32_bf16_vx},
    {"layer_normalization_u8_f16_vx", layer_normalization_u8_f16_vx},
    {"layer_normalization_wh_f16_vx", layer_normalization_wh_f16_vx},
    {"layer_normalization_wh_i16_vx", layer_normalization_wh_i16_vx},
    {"layer_normalization_wh_u8_vx", layer_normalization_wh_u8_vx},
    {"log_softmax_axis0_vx", log_softmax_axis0_vx},
    {"log_softmax_axis0_BF16_vx", log_softmax_axis0_BF16_vx},
    {"log_softmax_axis1_vx", log_softmax_axis1_vx},
    {"log_softmax_axis1_BF16_vx", log_softmax_axis1_BF16_vx},
    {"log_softmax_axis2_vx", log_softmax_axis2_vx},
    {"logical_not_vx", logical_not_vx},
    {"logical_ops_vx", logical_ops_vx},
    {"lstmunit_activation_BP_F16_vx", lstmunit_activation_BP_F16_vx},
    {"lstmunit_activation_BP_U8_vx", lstmunit_activation_BP_U8_vx},
    {"lstmunit_activation_B_F16_vx", lstmunit_activation_B_F16_vx},
    {"lstmunit_activation_B_U8_vx", lstmunit_activation_B_U8_vx},
    {"lstmunit_activation_CBP_F16_vx", lstmunit_activation_CBP_F16_vx},
    {"lstmunit_activation_CBP_U8_vx", lstmunit_activation_CBP_U8_vx},
    {"lstmunit_activation_CB_F16_vx", lstmunit_activation_CB_F16_vx},
    {"lstmunit_activation_CB_U8_vx", lstmunit_activation_CB_U8_vx},
    {"lstmunit_activation_CLP_F16_vx", lstmunit_activation_CLP_F16_vx},
    {"lstmunit_activation_CL_F16_vx", lstmunit_activation_CL_F16_vx},
    {"lstmunit_activation_CSP_F16_vx", lstmunit_activation_CSP_F16_vx},
    {"lstmunit_activation_CSP_U8_vx", lstmunit_activation_CSP_U8_vx},
    {"lstmunit_activation_CS_F16_vx", lstmunit_activation_CS_F16_vx},
    {"lstmunit_activation_CS_U8_vx", lstmunit_activation_CS_U8_vx},
    {"lstmunit_activation_LP_F16_vx", lstmunit_activation_LP_F16_vx},
    {"lstmunit_activation_L_F16_vx", lstmunit_activation_L_F16_vx},
    {"lstmunit_activation_SP_F16_vx", lstmunit_activation_SP_F16_vx},
    {"lstmunit_activation_SP_U8_vx", lstmunit_activation_SP_U8_vx},
    {"lstmunit_activation_S_F16_vx", lstmunit_activation_S_F16_vx},
    {"lstmunit_activation_S_U8_vx", lstmunit_activation_S_U8_vx},
    {"matrixmul_f16_vx", matrixmul_f16_vx},
    {"matrixmul_f16f16_u8_vx", matrixmul_f16f16_u8_vx},
    {"matrixmul_f16i16_i16_vx", matrixmul_f16i16_i16_vx},
    {"matrixmul_f16u8_f16_vx", matrixmul_f16u8_f16_vx},
    {"matrixmul_f16u8_u8_vx", matrixmul_f16u8_u8_vx},
    {"matrixmul_i16_vx", matrixmul_i16_vx},
    {"matrixmul_transA_vx", matrixmul_transA_vx},
    {"matrixmul_transB_f16_vx", matrixmul_transB_f16_vx},
    {"matrixmul_transB_f16_mix_vx", matrixmul_transB_f16_mix_vx},
    {"matrixmul_transB_u8_mix_vx", matrixmul_transB_u8_mix_vx},
    {"matrixmul_u8_vx", matrixmul_u8_vx},
    {"matrixmul_u8f16_f16_vx", matrixmul_u8f16_f16_vx},
    {"matrixmul_u8f16_u8_vx", matrixmul_u8f16_u8_vx},
    {"matrixmul_u8u8_f16_vx", matrixmul_u8u8_f16_vx},
    {"maximum_vx", maximum_vx},
    {"maximum_fp16_vx", maximum_fp16_vx},
    {"maximum_i16_vx", maximum_i16_vx},
    {"minimum_vx", minimum_vx},
    {"minimum_fp16_vx", minimum_fp16_vx},
    {"minimum_i16_vx", minimum_i16_vx},
    {"moments_axis0_vx", moments_axis0_vx},
    {"moments_axis01_vx", moments_axis01_vx},
    {"moments_axis012_vx", moments_axis012_vx},
    {"moments_axis1_vx", moments_axis1_vx},
    {"moments_axis2_vx", moments_axis2_vx},
    {"moments_u8_vx", moments_u8_vx},
    {"moments_u8_axis012_vx", moments_u8_axis012_vx},
    {"one_hot_vx", one_hot_vx},
    {"poolwithargmax_F16_vx", poolwithargmax_F16_vx},
    {"poolwithargmax_I16_vx", poolwithargmax_I16_vx},
    {"poolwithargmax_I8_vx", poolwithargmax_I8_vx},
    {"poolwithargmax_U8_vx", poolwithargmax_U8_vx},
    {"pow_fp16_vx", pow_fp16_vx},
    {"pow_fp16_i16_vx", pow_fp16_i16_vx},
    {"pow_fp16_i8_vx", pow_fp16_i8_vx},
    {"pow_i16_vx", pow_i16_vx},
    {"pow_i8_vx", pow_i8_vx},
    {"pow_u8_vx", pow_u8_vx},
    {"pre_process_bgra_vx", pre_process_bgra_vx},
    {"pre_process_gray_vx", pre_process_gray_vx},
    {"pre_process_gray_2_vx", pre_process_gray_2_vx},
    {"pre_process_gray_copy_vx", pre_process_gray_copy_vx},
    {"pre_process_nv12_scale_vx", pre_process_nv12_scale_vx},
    {"pre_process_nv12_scale_8bits_vx", pre_process_nv12_scale_8bits_vx},
    {"pre_process_nv12_scale_mix_vx", pre_process_nv12_scale_mix_vx},
    {"pre_process_rgb_vx", pre_process_rgb_vx},
    {"pre_process_rgb_copy_vx", pre_process_rgb_copy_vx},
    {"pre_process_yuv420_copy_u8_vx", pre_process_yuv420_copy_u8_vx},
    {"pre_process_yuv420_scale_fp16_vx", pre_process_yuv420_scale_fp16_vx},
    {"pre_process_yuv420_scale_i16_vx", pre_process_yuv420_scale_i16_vx},
    {"pre_process_yuv420_scale_i8_vx", pre_process_yuv420_scale_i8_vx},
    {"pre_process_yuv420_scale_u8_vx", pre_process_yuv420_scale_u8_vx},
    {"pre_process_yuv444_copy_u8_vx", pre_process_yuv444_copy_u8_vx},
    {"pre_process_yuv444_scale_vx", pre_process_yuv444_scale_vx},
    {"pre_process_yuv444_scale_fp16_vx", pre_process_yuv444_scale_fp16_vx},
    {"prelu_vx", prelu_vx},
    {"prelu_BF16_vx", prelu_BF16_vx},
    {"random_multinomial_vx", random_multinomial_vx},
    {"reduceall_internal_axis0_vx", reduceall_internal_axis0_vx},
    {"reduceall_internal_axis1_vx", reduceall_internal_axis1_vx},
    {"reduceall_internal_axis2_vx", reduceall_internal_axis2_vx},
    {"reduceany_internal_axis0_vx", reduceany_internal_axis0_vx},
    {"reduceany_internal_axis1_vx", reduceany_internal_axis1_vx},
    {"reduceany_internal_axis2_vx", reduceany_internal_axis2_vx},
    {"reducemax_internal_axis0_vx", reducemax_internal_axis0_vx},
    {"reducemax_internal_axis1_vx", reducemax_internal_axis1_vx},
    {"reducemax_internal_axis2_vx", reducemax_internal_axis2_vx},
    {"reducemin_internal_axis0_vx", reducemin_internal_axis0_vx},
    {"reducemin_internal_axis1_vx", reducemin_internal_axis1_vx},
    {"reducemin_internal_axis2_vx", reducemin_internal_axis2_vx},
    {"reduceprod_internal_axis0_vx", reduceprod_internal_axis0_vx},
    {"reduceprod_internal_axis1_vx", reduceprod_internal_axis1_vx},
    {"reduceprod_internal_axis2_vx", reduceprod_internal_axis2_vx},
    {"relational_ops_2d_vx", relational_ops_2d_vx},
    {"relational_ops_3d_vx", relational_ops_3d_vx},
    {"relu_keras_vx", relu_keras_vx},
    {"repeat_vx", repeat_vx},
    {"repeat_axis1_vx", repeat_axis1_vx},
    {"resize_1d_bilinear_BF16_vx", resize_1d_bilinear_BF16_vx},
    {"resize_1d_bilinear_DOWN_NX_vx", resize_1d_bilinear_DOWN_NX_vx},
    {"resize_1d_bilinear_F16_vx", resize_1d_bilinear_F16_vx},
    {"resize_1d_bilinear_I16_vx", resize_1d_bilinear_I16_vx},
    {"resize_1d_bilinear_I8_vx", resize_1d_bilinear_I8_vx},
    {"resize_1d_bilinear_U8_vx", resize_1d_bilinear_U8_vx},
    {"resize_1d_bilinear_U8_opt_vx", resize_1d_bilinear_U8_opt_vx},
    {"resize_1d_bilinear_UP_NX_vx", resize_1d_bilinear_UP_NX_vx},
    {"resize_1d_nearest_vx", resize_1d_nearest_vx},
    {"resize_bilinear_BF16_vx", resize_bilinear_BF16_vx},
    {"resize_bilinear_F16_vx", resize_bilinear_F16_vx},
    {"resize_bilinear_I16_vx", resize_bilinear_I16_vx},
    {"resize_bilinear_I8_vx", resize_bilinear_I8_vx},
    {"resize_bilinear_U8_vx", resize_bilinear_U8_vx},
    {"resize_bilinear_U8_half_pixel_centers_vx", resize_bilinear_U8_half_pixel_centers_vx},
    {"resize_bilinear_U8_opt_vx", resize_bilinear_U8_opt_vx},
    {"resize_bilinear_nhwc_vx", resize_bilinear_nhwc_vx},
    {"resize_nearest_vx", resize_nearest_vx},
    {"scatter_nd_vx", scatter_nd_vx},
    {"scatter_nd_big_vx", scatter_nd_big_vx},
    {"scatter_nd_update_vx", scatter_nd_update_vx},
    {"scatter_nd_update_atom_vx", scatter_nd_update_atom_vx},
    {"scatter_nd_update_big_vx", scatter_nd_update_big_vx},
    {"select_vx", select_vx},
    {"sequence_mask_vx", sequence_mask_vx},
    {"signal_frame_vx", signal_frame_vx},
    {"slice_vx", slice_vx},
    {"space2depth_internal_vx", space2depth_internal_vx},
    {"swish_vx", swish_vx},
    {"tensorstackconcat_vx", tensorstackconcat_vx},
    {"tile_vx", tile_vx},
    {"tile_mix_vx", tile_mix_vx},
    {"upsample_F16_vx", upsample_F16_vx},
    {"upsample_I16_vx", upsample_I16_vx},
    {"upsample_I8_vx", upsample_I8_vx},
    {"upsample_U8_vx", upsample_U8_vx},
    {"upsamplescale_vx", upsamplescale_vx},
    {"upsamplescale_k2_vx", upsamplescale_k2_vx},
    {"vsi_nn_kernel_header_vx", vsi_nn_kernel_header_vx},
    {"warp_affine_vx", warp_affine_vx},
};

static const source_map_t cl_resource[] =
{
    {"add_mean_std_norm_cl", add_mean_std_norm_cl},
    {"argmax_axis0_cl", argmax_axis0_cl},
    {"argmax_axis1_cl", argmax_axis1_cl},
    {"argmax_axis2_cl", argmax_axis2_cl},
    {"argmin_axis0_cl", argmin_axis0_cl},
    {"argmin_axis1_cl", argmin_axis1_cl},
    {"argmin_axis2_cl", argmin_axis2_cl},
    {"batchnorm_single_cl", batchnorm_single_cl},
    {"cast_cl", cast_cl},
    {"clip_F32_cl", clip_F32_cl},
    {"clip_U8_cl", clip_U8_cl},
    {"detect_post_box_cl", detect_post_box_cl},
    {"eltwise_ops_helper_cl", eltwise_ops_helper_cl},
    {"eltwise_unary_cl", eltwise_unary_cl},
    {"erf_cl", erf_cl},
    {"floordiv_cl", floordiv_cl},
    {"gather_cl", gather_cl},
    {"gather_nd_cl", gather_nd_cl},
    {"gather_nd_3d_cl", gather_nd_3d_cl},
    {"group_normalization_f32_cl", group_normalization_f32_cl},
    {"group_normalization_i32_cl", group_normalization_i32_cl},
    {"group_normalization_u8_cl", group_normalization_u8_cl},
    {"grucell_activation_cl", grucell_activation_cl},
    {"grucell_activation_sma_cl", grucell_activation_sma_cl},
    {"grucell_activation_z_h_cl", grucell_activation_z_h_cl},
    {"grucell_h_times_activation_r_cl", grucell_h_times_activation_r_cl},
    {"grucell_reset_after_activation_cl", grucell_reset_after_activation_cl},
    {"hswish_cl", hswish_cl},
    {"instance_normalization_f16_cl", instance_normalization_f16_cl},
    {"instance_normalization_f32_cl", instance_normalization_f32_cl},
    {"instance_normalization_i32_cl", instance_normalization_i32_cl},
    {"instance_normalization_u8_cl", instance_normalization_u8_cl},
    {"l2normalizescale_axis0_cl", l2normalizescale_axis0_cl},
    {"l2normalizescale_axis1_cl", l2normalizescale_axis1_cl},
    {"layer_normalization_cl", layer_normalization_cl},
    {"log_softmax_axis0_cl", log_softmax_axis0_cl},
    {"log_softmax_axis1_cl", log_softmax_axis1_cl},
    {"log_softmax_axis2_cl", log_softmax_axis2_cl},
    {"logical_not_cl", logical_not_cl},
    {"logical_ops_cl", logical_ops_cl},
    {"lstmunit_activation_BP_F32_cl", lstmunit_activation_BP_F32_cl},
    {"lstmunit_activation_BP_U8_cl", lstmunit_activation_BP_U8_cl},
    {"lstmunit_activation_B_F32_cl", lstmunit_activation_B_F32_cl},
    {"lstmunit_activation_B_U8_cl", lstmunit_activation_B_U8_cl},
    {"lstmunit_activation_CBP_F32_cl", lstmunit_activation_CBP_F32_cl},
    {"lstmunit_activation_CBP_U8_cl", lstmunit_activation_CBP_U8_cl},
    {"lstmunit_activation_CB_F32_cl", lstmunit_activation_CB_F32_cl},
    {"lstmunit_activation_CB_U8_cl", lstmunit_activation_CB_U8_cl},
    {"lstmunit_activation_CLP_F32_cl", lstmunit_activation_CLP_F32_cl},
    {"lstmunit_activation_CLP_U8_cl", lstmunit_activation_CLP_U8_cl},
    {"lstmunit_activation_CL_F32_cl", lstmunit_activation_CL_F32_cl},
    {"lstmunit_activation_CL_U8_cl", lstmunit_activation_CL_U8_cl},
    {"lstmunit_activation_CSP_F32_cl", lstmunit_activation_CSP_F32_cl},
    {"lstmunit_activation_CSP_U8_cl", lstmunit_activation_CSP_U8_cl},
    {"lstmunit_activation_CS_F32_cl", lstmunit_activation_CS_F32_cl},
    {"lstmunit_activation_CS_U8_cl", lstmunit_activation_CS_U8_cl},
    {"lstmunit_activation_LP_F32_cl", lstmunit_activation_LP_F32_cl},
    {"lstmunit_activation_L_F32_cl", lstmunit_activation_L_F32_cl},
    {"lstmunit_activation_SP_F32_cl", lstmunit_activation_SP_F32_cl},
    {"lstmunit_activation_SP_U8_cl", lstmunit_activation_SP_U8_cl},
    {"lstmunit_activation_S_F32_cl", lstmunit_activation_S_F32_cl},
    {"lstmunit_activation_S_U8_cl", lstmunit_activation_S_U8_cl},
    {"matrixmul_cl", matrixmul_cl},
    {"matrixmul_transA_cl", matrixmul_transA_cl},
    {"maximum_cl", maximum_cl},
    {"minimum_cl", minimum_cl},
    {"moments_axis0_cl", moments_axis0_cl},
    {"moments_axis01_cl", moments_axis01_cl},
    {"moments_axis012_cl", moments_axis012_cl},
    {"moments_axis1_cl", moments_axis1_cl},
    {"moments_axis2_cl", moments_axis2_cl},
    {"one_hot_cl", one_hot_cl},
    {"poolwithargmax_cl", poolwithargmax_cl},
    {"pow_cl", pow_cl},
    {"prelu_cl", prelu_cl},
    {"random_multinomial_cl", random_multinomial_cl},
    {"reduceall_internal_axis0_cl", reduceall_internal_axis0_cl},
    {"reduceall_internal_axis1_cl", reduceall_internal_axis1_cl},
    {"reduceall_internal_axis2_cl", reduceall_internal_axis2_cl},
    {"reduceany_internal_axis0_cl", reduceany_internal_axis0_cl},
    {"reduceany_internal_axis1_cl", reduceany_internal_axis1_cl},
    {"reduceany_internal_axis2_cl", reduceany_internal_axis2_cl},
    {"reducemax_internal_axis0_cl", reducemax_internal_axis0_cl},
    {"reducemax_internal_axis1_cl", reducemax_internal_axis1_cl},
    {"reducemax_internal_axis2_cl", reducemax_internal_axis2_cl},
    {"reducemin_internal_axis0_cl", reducemin_internal_axis0_cl},
    {"reducemin_internal_axis1_cl", reducemin_internal_axis1_cl},
    {"reducemin_internal_axis2_cl", reducemin_internal_axis2_cl},
    {"reduceprod_internal_axis0_cl", reduceprod_internal_axis0_cl},
    {"reduceprod_internal_axis1_cl", reduceprod_internal_axis1_cl},
    {"reduceprod_internal_axis2_cl", reduceprod_internal_axis2_cl},
    {"relational_ops_cl", relational_ops_cl},
    {"relu_keras_cl", relu_keras_cl},
    {"repeat_cl", repeat_cl},
    {"resize_1d_bilinear_cl", resize_1d_bilinear_cl},
    {"resize_1d_nearest_cl", resize_1d_nearest_cl},
    {"resize_bilinear_cl", resize_bilinear_cl},
    {"resize_nearest_cl", resize_nearest_cl},
    {"roi_align_cl", roi_align_cl},
    {"scatter_nd_cl", scatter_nd_cl},
    {"scatter_nd_update_cl", scatter_nd_update_cl},
    {"select_cl", select_cl},
    {"sequence_mask_cl", sequence_mask_cl},
    {"signal_frame_cl", signal_frame_cl},
    {"slice_cl", slice_cl},
    {"space2depth_internal_cl", space2depth_internal_cl},
    {"swish_cl", swish_cl},
    {"tile_cl", tile_cl},
    {"upsample_cl", upsample_cl},
};

static const char* _load_code
    (
    const char* source_name,
    size_t* size,
    const source_map_t* source_map,
    size_t source_map_size,
    const char* tail
    )
{
    const char* source;
    char source_path[VSI_NN_MAX_PATH];
    size_t n;
    int i;
    source = NULL;
    n = snprintf( source_path, VSI_NN_MAX_PATH, "%s%s", source_name, tail );
    if( n == VSI_NN_MAX_PATH )
    {
        VSILOGE("Kernel source path overflow %d/%d", n, VSI_NN_MAX_PATH);
        *size = 0;
        return NULL;
    }
    for( i = 0; i < (int)source_map_size; i++ )
    {
        if( strncmp( source_map[i].name, source_path, VSI_NN_MAX_PATH ) == 0 )
        {
            source = source_map[i].data;
            *size = strlen( source );
            break;
        }
    }
    if( !source )
    {
        *size = 0;
    }
    return source;
} /* _load_code() */

const char* vsi_nn_resource_load_source_code
    (
    const char* source_name,
    size_t* size,
    vsi_nn_kernel_type_e type
    )
{
    const char* s = NULL;
    switch( type )
    {
        case VSI_NN_KERNEL_TYPE_EVIS:
            s = _load_code( source_name, size,
                evis_resource, _cnt_of_array(evis_resource), "_vx" );
            break;
        case VSI_NN_KERNEL_TYPE_CL:
            s = _load_code( source_name, size,
                cl_resource, _cnt_of_array(cl_resource), "_cl" );
            break;
        default:
            break;
    }
    return s;
} /* vsi_nn_resource_load_source_code() */
