TOSA 0.80.0 specification

bool tosa_test_compliance(tosa_graph_t graph, tosa_list_t input_list, tosa_level_t level) {
    shape_list_t output_list_spec = tosa_allocate_list(tosa_output_shape(graph));
    shape_list_t output_list_test = tosa_allocate_list(tosa_output_shape(graph));
    tosa_graph_result = tosa_valid;    // result starts as valid
    tosa_nesting_depth = 0;            // if/while nesting level
    tosa_execute_graph(graph, input_list, output_list_spec, level);
    if (tosa_graph_result == tosa_unpredictable) {
        return true;    // No requirement to match an unpredictable result
    }
    result_test = execute_implementation_under_test(graph, input_list, output_list_test);
    if (tosa_graph_result == tosa_error) {
        return result_test == tosa_error;   // result must be an error
    }
    if (exact_tensor_match(output_list_spec, output_list_test)) {
       // Predictable bit-exact value match required
       return true;
    }
    return false;
}

int32_t apply_scale_32(int32_t value, int32_t multiplier, int8_t shift, bool_t double_round=false) {
    REQUIRE(multiplier >= 0);
    REQUIRE(2 <= shift && shift <= 62);
    REQUIRE(value >= (-1 << (shift - 1)) && value < (1 << (shift - 1)));
    int64_t round = 1 << (shift - 1);
    if (double_round) {
        if (shift > 31 && value >= 0) round += 1<<30;
        if (shift > 31 && value < 0)  round -= 1<<30;
    }
    int64_t result = static_cast<int64_t>(value) * multiplier + round;
    result = result >> shift;
    // result will fit a 32-bit range due to the REQUIRE on value
    return static_cast<int32_t>(result);
}

int32_t apply_scale_16(int48_t value, int16_t multipler, int8_t shift) {
    REQUIRE(multiplier >= 0);
    REQUIRE(2 <= shift && shift <= 62);
    int64_t round = (1 << (shift - 1));
    int64_t result = static_cast<int64_t>(value) * multiplier + round;
    result = result >> shift;
    REQUIRE(result >= minimum<int32_t> && result <= maximum<int32_t>);
    return static_cast<int32_t>(result);
}

typedef struct {
    int32_t multiplier;
    int8_t shift;
} scale_t;

scale_t reciprocal_scale(uint32_t value) {
    REQUIRE(value > 0);
    scale_t scale;
    int32_t k = 32 - count_leading_zeros(value - 1); // (1 << k) / 2 < value <= (1 << k)
    int64_t numerator = ((1 << 30) + 1) << k;
    scale.multiplier = numerator / value; // (1 << 30) <= multiplier < (1 << 31)
    scale.shift = 30 + k;
    return scale;
}

int32_t apply_lookup_s(int16_t *table, int32_t value)
{
    int16_t clipped_value = static_cast<int16_t>(apply_clip_s<int32_t>(value, -32768, +32767));
    int32_t index = (clipped_value + 32768) >> 7;
    int32_t fraction = clipped_value & 0x7f;
    int16_t base = table[index];
    int16_t next = table[index+1];
    int32_t slope = next - base;
    REQUIRE(slope >= minimum<int16_t> && slope <= maximum<int16_t>)
    int32_t return_value = (base << 7) + slope * fraction;
    return return_value;	// return interpolated value of 16 + 7 = 23 bits
}

void generate_lookup_table(int16_t *table, int32_t (*reference)(int32_t))
{
    for (int i = -256; i <= 256; i++) {
        int32_t value = (*reference)(i);
        table[i + 256] = static_cast<int16_t>(apply_clip<int32_t>(value, -32768, +32767));
    }
}

LEVEL_CHECK(rank(shape1) <= MAX_RANK);

ERROR_IF(axis < 0 || axis >= rank(shape1));
if (axis == 0) {
    left_shape = [];
} else {
    left_shape = shape1[0:axis - 1];
}
if (axis == rank(shape1)-1) {
    right_shape = [];
} else {
    right_shape = shape1[axis+1:rank(shape1) - 1];
}
ERROR_IF(flatten(left_shape, right_shape) != shape);
for_each(left_index in left_shape) {
    for_each(right_index in right_shape) {
        in_t max_value = minimum_s<in_t>;
        out_t max_index = 0;
        for (i = 0; i < shape[axis]; i++) {
            dim_t index = flatten(left_index, [i], right_index);
            in_t value = tensor_read<in_t>(input, shape1, index);
            if (apply_max_s<in_t>(value, max_value) != max_value) {
                max_value = value;
                max_index = i;
            }
        }
        dim_t index = flatten(left_index, right_index);
        tensor_write<out_t>(output, shape, index, max_index);
    }
}

LEVEL_CHECK(kernel_y <= MAX_KERNEL);
LEVEL_CHECK(kernel_x <= MAX_KERNEL);
LEVEL_CHECK(stride_y <= MAX_STRIDE);
LEVEL_CHECK(stride_x <= MAX_STRIDE);
LEVEL_CHECK(pad_top <= MAX_KERNEL);
LEVEL_CHECK(pad_bottom <= MAX_KERNEL);
LEVEL_CHECK(pad_left <= MAX_KERNEL);
LEVEL_CHECK(pad_right <= MAX_KERNEL);

ERROR_IF(in_out_t != i8_t && input_zp != 0); // Zero point only for int8_t
ERROR_IF(in_out_t != i8_t && output_zp != 0); // Zero point only for int8_t
ERROR_IF(kernel_y < 1 || kernel_x < 1); // kernel size must be >= 1
ERROR_IF(stride_y < 1 || stride_x < 1);
ERROR_IF(pad_top < 0 || pad_bottom < 0 || pad_left < 0 || pad_right < 0);
// Padding must be less than kernel size to avoid
// a divide-by-zero.
ERROR_IF(pad_right >= kernel_x || pad_left >= kernel_x);
ERROR_IF(pad_top >= kernel_y || pad_bottom >= kernel_y);
ERROR_IF(OH != idiv_check(IH + pad_top + pad_bottom - kernel_y, stride_y) + 1);
ERROR_IF(OW != idiv_check(IW + pad_left + pad_right - kernel_x, stride_x) + 1);

for_each(0 <= n < N, 0 <= oy < OH, 0 <= ox < OW, 0 <= c < C ) {
    in_out_t output_val;
    acc_t acc = 0;
    int count = 0;
    index_t iy = oy * stride_y - pad_top;
    index_t ix = ox * stride_x - pad_left;
    for_each(0 <= ky < kernel_y, 0 <= kx < kernel_x) {
        index_t y = iy + ky;
        index_t x = ix + kx;
        // Only values from the input tensor are used to calculate the
        // average, padding does not count
        if (0 <= y < IH and 0 <= x < IW) {
            count++;
            acc_t value = sign_extend<acc_t>(tensor_read<in_out_t>(input, [N,IH,IW,C], [n,y,x,c]));
            value = apply_sub_s<acc_t>(value, sign_extend<acc_t>(input_zp));
            acc = apply_add_s<acc_t>(acc, value);
        }
    }
    if (is_float(in_out_t)) {
        output_val = acc / static_cast<in_out_t>(count);
    } else {
        scale_t scale = reciprocal_scale(count);
        acc = apply_scale_32(acc, scale.multiplier, scale.shift, false);
        acc = apply_add_s<acc_t>(acc, sign_extend<acc_t>(output_zp));
        acc = apply_clip_s<acc_t>(acc, minimum_s<in_out_t>, maximum_s<in_out_t>);
        output_val = static_cast<in_out_t>(acc);
    }
    tensor_write<in_out_t>(output, [N,OH,OW,C], [n,oy,ox,c], output_val);
}

LEVEL_CHECK(dilation_y * KH <= MAX_KERNEL);
LEVEL_CHECK(dilation_x * KW <= MAX_KERNEL);
LEVEL_CHECK(pad_top <= MAX_KERNEL);
LEVEL_CHECK(pad_bottom <= MAX_KERNEL);
LEVEL_CHECK(pad_left <= MAX_KERNEL);
LEVEL_CHECK(pad_right <= MAX_KERNEL);
LEVEL_CHECK(stride_y <= MAX_STRIDE);
LEVEL_CHECK(stride_x <= MAX_STRIDE);

ERROR_IF(in_t != i8_t && input_zp != 0); // Zero point only for int8_t
ERROR_IF(weight_t != int8_t && weight_zp != 0);
ERROR_IF(pad_top < 0 || pad_bottom < 0 || pad_left < 0 || pad_right < 0);
ERROR_IF(stride_y < 1 || stride_x < 1);
ERROR_IF(dilation_y < 1 || dilation_x < 1);
ERROR_IF(OH != idiv_check(IH - 1 + pad_top + pad_bottom - (KH - 1) * dilation_y, stride_y) + 1);
ERROR_IF(OW != idiv_check(IW - 1 + pad_left + pad_right - (KW - 1) * dilation_x, stride_x) + 1);
ERROR_IF(BC != OC && BC != 1);

for_each(0 <= n < N, 0 <= oy < OH, 0 <= ox < OW; 0 <= oc < OC) {
    out_t acc = 0;
    index_t iy = oy * stride_y - pad_top;
    index_t ix = ox * stride_x - pad_left;
    for_each(0 <= ky < KH, 0 <= kx < KW, 0 <= ic < IC) {
        index_t y = iy + ky * dilation_y;
        index_t x = ix + kx * dilation_x;
        if (0 <= y < IH && 0 <= x < IW) {
            out_t value  = static_cast<out_t>(tensor_read<in_t>(input,
                                                                [N,IH,IW,IC],
                                                                [n,y,x,ic]));
            out_t weight = static_cast<out_t>(tensor_read<weight_t>(weight,
                                                                   [OC,KH,KW,IC],
                                                                   [oc,ky,kx,ic]));
            value  = apply_sub_s<out_t>(value, static_cast<out_t>(input_zp));
            weight = apply_sub_s<out_t>(weight, static_cast<out_t>(weight_zp));
            acc = apply_add_s<out_t>(acc, apply_mul_s<out_t>(value, weight));
        }
    }
    acc = apply_add_s<out_t>(acc, bias[(BC == 1) ? 0 : oc]);
    tensor_write<out_t>(output, [N,OH,OW,OC], [n,oy,ox,oc], acc);
}

LEVEL_CHECK(dilation_d * KD <= MAX_KERNEL);
LEVEL_CHECK(dilation_y * KH <= MAX_KERNEL);
LEVEL_CHECK(dilation_x * KW <= MAX_KERNEL);
LEVEL_CHECK(pad_d0 <= MAX_KERNEL);
LEVEL_CHECK(pad_d1 <= MAX_KERNEL);
LEVEL_CHECK(pad_top <= MAX_KERNEL);
LEVEL_CHECK(pad_bottom <= MAX_KERNEL);
LEVEL_CHECK(pad_left <= MAX_KERNEL);
LEVEL_CHECK(pad_right <= MAX_KERNEL);
LEVEL_CHECK(stride_y <= MAX_STRIDE);
LEVEL_CHECK(stride_x <= MAX_STRIDE);
LEVEL_CHECK(stride_d <= MAX_STRIDE);

ERROR_IF(in_t != i8_t && input_zp != 0); // Zero point only for int8_t
ERROR_IF(weight_t != i8_t && weight_zp != 0);
ERROR_IF(pad_d0 < 0 || pad_d1 < 0 || pad_top < 0 || pad_bottom < 0 || pad_left < 0 || pad_right < 0);
ERROR_IF(stride_d < 1 || stride_y < 1 || stride_x < 1);
ERROR_IF(dilation_d < 1 || dilation_y < 1 || dilation_x < 1);
ERROR_IF(OD != idiv_check(ID - 1 + pad_d0 + pad_d1      - (KD - 1) * dilation_d, stride_d) + 1);
ERROR_IF(OH != idiv_check(IH - 1 + pad_top + pad_bottom - (KH - 1) * dilation_y, stride_y) + 1);
ERROR_IF(OW != idiv_check(IW - 1 + pad_left + pad_right - (KW - 1) * dilation_x, stride_x) + 1);
ERROR_IF(BC != OC && BC != 1);

for_each(0 <= n < N, 0 <= od < OD, 0 <= oy < OH, 0 <= ox < OW; 0 <= oc < OC) {
    out_t acc = 0;
    index_t id = od * stride_d - pad_d0;
    index_t iy = oy * stride_y - pad_top;
    index_t ix = ox * stride_x - pad_left;
    for_each(0 <= kd < KD, 0 <= ky < KH, 0 <= kx < KW, 0 <= ic < IC) {
        index_t d = id + kd * dilation_d;
        index_t y = iy + ky * dilation_y;
        index_t x = ix + kx * dilation_x;
        if (0 <= x < IW && 0 <= y < IH && 0 <= d < ID) {
            out_t value  = static_cast<out_t>(tensor_read<in_t>(input,
                                                                [N,ID,IH,IW,IC],
                                                                [n,d,y,x,ic]));
            out_t weight = static_cast<out_t>(tensor_read<weight_t>(weight,
                                                                    [OC,KD,KH,KW,IC],
                                                                    [oc,kd,ky,kx,ic]));
            value  = apply_sub_s<out_t>(value, static_cast<out_t>(input_zp));
            weight = apply_sub_s<out_t>(weight, static_cast<out_t>(weight_zp));
            acc = apply_add_s<out_t>(acc, apply_mul_s<out_t>(value, weight));
        }
    }
    acc = apply_add_s<out_t>(acc, bias[(BC == 1) ? 0 : oc]);
    tensor_write<out_t>(output, [N,OD,OH,OW,OC], [n,od,oy,ox,oc], acc);
}

LEVEL_CHECK(dilation_y * KH <= MAX_KERNEL);
LEVEL_CHECK(dilation_x * KW <= MAX_KERNEL);
LEVEL_CHECK(pad_top <= MAX_KERNEL);
LEVEL_CHECK(pad_bottom <= MAX_KERNEL);
LEVEL_CHECK(pad_left <= MAX_KERNEL);
LEVEL_CHECK(pad_right <= MAX_KERNEL);
LEVEL_CHECK(stride_y <= MAX_STRIDE);
LEVEL_CHECK(stride_x <= MAX_STRIDE);

ERROR_IF(in_t != i8_t && input_zp != 0); // Zero point only for int8_t
ERROR_IF(weight_t != i8_t && weight_zp != 0);
ERROR_IF(pad_top < 0 || pad_bottom < 0 || pad_left < 0 || pad_right < 0);
ERROR_IF(stride_y < 1 || stride_x < 1);
ERROR_IF(dilation_y < 1 || dilation_x < 1);
ERROR_IF(OH != idiv_check(IH - 1 + pad_top + pad_bottom - (KH - 1) * dilation_y, stride_y) + 1);
ERROR_IF(OW != idiv_check(IW - 1 + pad_left + pad_right - (KW - 1) * dilation_x, stride_x) + 1);
ERROR_IF(BC != C*M && BC != 1);

for_each(0 <= n < N, 0 <= oy < OH, 0 <= ox < OW; 0 <= c < C, 0 <= m < M) {
    out_t acc = 0;
    index_t iy = oy * stride_y - pad_top;
    index_t ix = ox * stride_x - pad_left;
    for_each(0 <= ky < KH, 0 <= kx < KW) {
        index_t y = iy + ky * dilation_y;
        index_t x = ix + kx * dilation_x;
        if (0 <= y < IH && 0 <= x < IW) {
            out_t value  = static_cast<out_t>(tensor_read<in_t>(input,
                                                                [N,IH,IW,C],
                                                                [n,y,x,c]));
            out_t weight = static_cast<out_t>(tensor_read<weight_t>(weight,
                                                                    [KH,KW,C,M],
                                                                    [ky,kx,c,m]));
            value  = apply_sub_s<out_t>(value, static_cast<out_t>input_zp);
            weight = apply_sub_s<out_t>(weight, static_cast<out_t>weight_zp);
            acc = apply_add_s<out_t>(acc, apply_mul_s<out_t>(value, weight));
        }
    }
    acc = apply_add_s<out_t>(acc, bias[(BC == 1) ? 0 : (c * M) + m]);
    tensor_write<out_t>(output, [N,OH,OW,C * M], [n,oy,ox,c * M + m], acc);
}

LEVEL_CHECK(H <= MAX_KERNEL);
LEVEL_CHECK(W <= MAX_KERNEL);

ERROR_IF(!power_of_two(H));
ERROR_IF(!power_of_two(W));

float sign_val = 1.0;

if (inverse) {
    sign_val = -1.0;
}

for_each(0 <= n < N, 0 <= oy < H, 0 <= ox < W) {
    in_out_t sum_real = 0.0;
    in_out_t sum_imag = 0.0;
    for_each(0 <= iy < H, 0 <= ix < W) {
        in_out_t val_real = tensor_read<in_out_t>(input_real, [N,H,W], [n,iy,ix]);
        in_out_t val_imag = tensor_read<in_out_t>(input_imag, [N,H,W], [n,iy,ix]);
        float_t a = sign_val * 2 * pi() * ((iy * oy) / H + (ix * ox) / W);
        sum_real += val_real * cos(a) + val_imag * sin(a);
        sum_imag += -val_real * sin(a) + val_imag * cos(a);
    }
    tensor_write<in_out_t>(output_real, [N,H,W], [n,oy,ox], sum_real);
    tensor_write<in_out_t>(output_imag, [N,H,W], [n,oy,ox], sum_imag);
}

ERROR_IF(in_t != i8_t && input_zp != 0); // Zero point only for int8_t
ERROR_IF(weight_t != i8_t && weight_zp != 0);
ERROR_IF(BC != OC && BC != 1);

for_each(0 <= n < N, 0 <= oc < OC) {
    out_t acc = 0;
    for_each(0 <= ic < IC) {
        out_t value  = static_cast<out_t>(tensor_read<in_t>(input, [N,IC], [n,ic]));
        out_t weight = static_cast<out_t>(tensor_read<weight_t>(weight, [OC,IC], [oc,ic]));
        value  = apply_sub_s<out_t>(value, static_cast<out_t>(input_zp));
        weight = apply_sub_s<out_t>(weight, static_cast<out_t>(weight_zp));
        acc = apply_add_s<out_t>(acc, apply_mul_s<out_t>(value, weight));
    }
    acc = apply_add_s<out_t>(acc, bias[(BC == 1) ? 0 : oc]);
    tensor_write<out_t>(output, [N,OC], [n,oc], acc);
}

ERROR_IF(in_t != i8_t && (A_zp != 0 || B_zp != 0)); // Zero point only for int8_t
for_each(0 <= n < N, 0 <= h < H, 0 <= w < W) {
    out_t acc = 0;
    for_each(0 <= c < C) {
        out_t value1 = static_cast<out_t>(tensor_read<in_t>(A, [N,H,C], [n,h,c]));
        out_t value2 = static_cast<out_t>(tensor_read<in_t>(B, [N,C,W], [n,c,w]));
        value1 = apply_sub_s<out_t>(value1, static_cast<out_t>(A_zp));
        value2 = apply_sub_s<out_t>(value2, static_cast<out_t>(B_zp));
        acc = apply_add_s<out_t>(acc, apply_mul_s<out_t>(value1 * value2));
    }
    tensor_write<out_t>(output, [N,H,W], [n,h,w], acc);
}

LEVEL_CHECK(kernel_y <= MAX_KERNEL);
LEVEL_CHECK(kernel_x <= MAX_KERNEL);
LEVEL_CHECK(stride_y <= MAX_STRIDE);
LEVEL_CHECK(stride_x <= MAX_STRIDE);
LEVEL_CHECK(pad_top <= MAX_KERNEL);
LEVEL_CHECK(pad_bottom <= MAX_KERNEL);
LEVEL_CHECK(pad_left <= MAX_KERNEL);
LEVEL_CHECK(pad_right <= MAX_KERNEL);

ERROR_IF(kernel_y < 1 || kernel_x < 1); // kernel size must be >= 1
ERROR_IF(stride_y < 1 || stride_x < 1);
ERROR_IF(pad_top < 0 || pad_bottom < 0 || pad_left < 0 || pad_right < 0);
// Padding must be less than kernel size, otherwise no
// input values will be used.
ERROR_IF(pad_right >= kernel_x || pad_left >= kernel_x);
ERROR_IF(pad_top >= kernel_y || pad_bottom >= kernel_y);
ERROR_IF(OH != idiv_check(IH + pad_top + pad_bottom - kernel_y, stride_y) + 1);
ERROR_IF(OW != idiv_check(IW + pad_left + pad_right - kernel_x, stride_x) + 1);

for_each(0 <= n < N, 0 <= oy < H, 0 <= ox < W, 0 <= c < C ) {
    in_out_t acc = minimum_value<in_out_t>;
    index_t iy = oy * stride_y - pad_top;
    index_t ix = ox * stride_x - pad_left;
    for_each( 0 <= ky < kernel_y, 0 <= kx < kernel_x ) {
        index_t y = iy + ky;
        index_t x = ix + kx;
        if (y >= 0 && y < IH && x >= 0 && x < IW) {
            in_out_t value = tensor_read<in_out_t>(input, [N,IH,IW,C], [n,y,x,c]);
            acc = apply_max_s<in_out_t>(acc, value);
        }
    }
    tensor_write<in_out_t>(output, [N,OH,OW,C], [n,oy,ox,c], acc);
}

LEVEL_CHECK(H <= MAX_KERNEL);
LEVEL_CHECK(W <= MAX_KERNEL);

ERROR_IF(!power_of_two(H));
ERROR_IF(!power_of_two(W));

for_each(0 <= n < N, 0 <= oy < H, 0 <= ox < W/2 + 1) {
    in_out_t sum_real = 0.0;
    in_out_t sum_imag = 0.0;
    for_each(0 <= iy < H, 0 <= ix < W) {
        in_out_t val_real = tensor_read<in_out_t>(input_real, [N,H,W], [n,iy,ix]);
        float_t a = 2 * pi() * ((iy * oy) / H + (ix * ox) / W);
        sum_real += val_real * cos(a);
        sum_imag += -val_real * sin(a);
    }
    tensor_write<in_out_t>(output_real, [N,H,W], [n,oy,ox], sum_real);
    tensor_write<in_out_t>(output_imag, [N,H,W], [n,oy,ox], sum_imag);
}

LEVEL_CHECK(KH <= MAX_KERNEL);
LEVEL_CHECK(KW <= MAX_KERNEL);
LEVEL_CHECK(out_pad_top <= MAX_KERNEL);
LEVEL_CHECK(out_pad_bottom <= MAX_KERNEL);
LEVEL_CHECK(out_pad_left <= MAX_KERNEL);
LEVEL_CHECK(out_pad_right <= MAX_KERNEL);
LEVEL_CHECK(stride_y <= MAX_STRIDE);
LEVEL_CHECK(stride_x <= MAX_STRIDE);

ERROR_IF(in_t != i8_t  && input_zp != 0); // Zero point only allowed for int8_t
ERROR_IF(weight_t != i8_t && weight_zp != 0);
ERROR_IF(out_pad_top <= -KH || out_pad_bottom <= -KH);
ERROR_IF(out_pad_left <= -KW || out_pad_right <= -KW);
ERROR_IF(stride_y < 1 || stride_x < 1);
ERROR_IF(OH != (IH - 1) * stride_y + out_pad_top + out_pad_bottom + KH);
ERROR_IF(OW != (IW - 1) * stride_x + out_pad_left + out_pad_right + KW);
ERROR_IF(BC != OC && BC != 1);

for_each(index in out_shape) {
    tensor_write<out_t>(output, [N,OH,OW,OC], index, bias[(BC == 1) ? 0 : index[3]])
}
for_each(0 <= n < N, 0 <= iy < IH, 0 <= ix < IW, 0 <= oc < OC,
          0 <= ic < IC, 0 <= ky < KH,  0 <= kx < KW) {
    index_t oy = iy * stride_y + out_pad_top + ky;
    index_t ox = ix * stride_x + out_pad_left + kx;
    if (oy >= 0 && oy < OH && ox >= 0 && ox < OW) {
        out_t acc = static_cast<out_t>(tensor_read<out_t>(output, [N,OH,OW,OC], [n,oy,ox,oc]));
        out_t value = static_cast<out_t>(tensor_read<in_t>(input, [N,IH,IW,IC], [n,iy,ix,ic]));
        out_t weight = static_cast<out_t>(tensor_read<weight_t>(weight, [OC,KH,KW,IC], [oc,ky,kx,ic]));
        value = apply_sub_s<out_t>(value, static_cast<out_t>(input_zp));
        weight = apply_sub_s<out_t>(weight, static_cast<out_t>(weight_zp));
        acc = apply_add_s<out_t>(acc, apply_mul_s<out_t>(value, weight));
        tensor_write<out_t>(output, [N,OH,OW,OC], [n,oy,ox,oc], acc);
    }
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(max_val < min_val);
for_each(index in shape) {
    in_out_t value = tensor_read<in_out_t>(input, shape, index);
    value = apply_clip<in_out_t>(value, min_val, max_val);
    tensor_write<in_out_t>(output, shape, index, value);
}

int16_t erf_reference(int16_t x) { // input x range is -256 to + 256 inclusive
    F64 v = (double)x / (double)64;
    v = erf(v);
    return round_to_nearest_int(32768.0 * v);
}

generate_lookup_table(&erf_table, &erf_reference);

LEVEL_CHECK(rank(shape) <= MAX_RANK);

int16_t sigmoid_reference(int16_t x) { // input x range is -256 to + 256 inclusive
    fp64_t v = (fp64_t)x / (fp64_t)16;
    v = 1.0/(1.0 + exp(-v));
    return round_to_nearest_int(32768.0 * v);
}

generate_lookup_table(&sigmoid_table, &sigmoid_reference);

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input, shape, index);
    value = sigmoid<in_out_t>(value1);
    tensor_write<in_out_t>(output, shape, index, value);
}

int16_t tanh_reference(int16_t x) {  // input x range is -256 to +256 inclusive
    fp64_t v = (fp64_t)x/(fp64_t)32;
    v = exp(-2.0*v);
    v = (1.0-v)/(1.0+v);
    return round_to_nearest_int(32768.0 * v);
}

generate_lookup_table(&tanh_table, &tanh_reference);

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input, shape, index);
    value = tanh<in_out_t>(value1);
    tensor_write<in_out_t>(output, shape, index, value);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

if (in_out_t == shape_t) {
    ERROR_IF(rank(shape) != 0 || rank(shape1) != 0 || rank(shape2) != 0);
    shape_t value1 = tensor_read<shape_t>(input1, [], []);
    shape_t value2 = tensor_read<shape_t>(input2, [], []);
    shape_t result = apply_add_s<shape_t>(value1, value2);
    tensor_write<shape_t>(output, [], [], result);
} else {
    ERROR_IF(shape != broadcast_shape(shape1, shape2));
    for_each(index in shape) {
        dim_t index1 = apply_broadcast(shape, shape1, index);
        dim_t index2 = apply_broadcast(shape, shape2, index);
        in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
        in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
        in_out_t result = apply_add_s<in_out_t>(value1, value2);
        tensor_write<in_out_t>(output, shape, index, result);
    }
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);

    // Ensure that shift amount is appropriate for the data type
    REQUIRE((in_out_t == i32_t && 0 <= value2 && value2 <= 31) ||
            (in_out_t == i16_t && 0 <= value2 && value2 <= 15) ||
            (in_out_t == i8_t && 0 <= value2 && value2 <= 7));

    in_out_t result = apply_arith_rshift<in_out_t>(value1, value2);
    if (round == true && static_cast<int32_t>(value2) > 0 &&
        (apply_arith_rshift<in_out_t>(value1, apply_sub_s<in_out_t>(value2, 1)) & 1 != 0) {
        result = result + 1;
    }
    result = apply_clip_s<in_out_t>(result, minimum_s<in_out_t>, maximum_s<in_out_t>);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = value1 & value2;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = value1 | value2;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = value1 ^ value2;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

if (in_out_t == shape_t) {
    ERROR_IF(rank(shape) != 0 || rank(shape1) != 0 || rank(shape2) != 0);
    shape_t value1 = tensor_read<shape_t>(input1, [], []);
    shape_t value2 = tensor_read<shape_t>(input2, [], []);
    REQUIRE(value2 != 0);
    shape_t result = value1 / value2;
    tensor_write<shape_t>(output, [], [], result);
} else {
    ERROR_IF(shape != broadcast_shape(shape1, shape2));
    for_each(index in shape) {
        dim_t index1 = apply_broadcast(shape, shape1, index);
        dim_t index2 = apply_broadcast(shape, shape2, index);
        in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
        in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
        REQUIRE(value2 != 0);
        // This catches the case where we divide minimum<in_out_t> by -1
        // which is not representable in two's complement
        REQUIRE(static_cast<int64_t>(value1) / static_cast<int64_t>(value2) <= maximum_s<in_out_t>);
        in_out_t result = apply_intdiv_s<in_out_t>(value1, value2);
        tensor_write<in_out_t>(output, shape, index, result);
    }
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = value1 && value2;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    REQUIRE(0 <= value2 && value2 <= 31);
    in_out_t result = value1 << value2;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    REQUIRE(0 <= static_cast<int32_t>(value2) && static_cast<int32_t>(value2) <= 31);
    // Logical shifts happen as unsigned types internally
    in_out_t result = apply_logical_rshift<in_out_t>(value1, value2);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = value1 || value2;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = value1 != value2;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = apply_max_s<in_out_t>(value1, value2);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = apply_min_s(value1, value2);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

if (in_out_t == shape_t) {
    ERROR_IF(rank(shape) != 0 || rank(shape1) != 0 || rank(shape2) != 0);
    shape_t value1 = tensor_read<shape_t>(input1, [], []);
    shape_t value2 = tensor_read<shape_t>(input2, [], []);
    shape_t result = value1 * value2;
    tensor_write<shape_t>(output, [], [], result);
} else {
    REQUIRE(0 <= shift && shift <= 63);
    REQUIRE(in_t == int32_t || shift == 0);
    ERROR_IF(shape != broadcast_shape(shape1, shape2));
    for_each(index in shape) {
        dim_t index1 = apply_broadcast(shape, shape1, index);
        dim_t index2 = apply_broadcast(shape, shape2, index);
        in_t value1 = tensor_read<in_t>(input1, shape1, index1);
        in_t value2 = tensor_read<in_t>(input2, shape2, index2);
        out_t result;
        if (in_t == i32_t && shift > 0) {
            int64_t product = sign_extend<int64_t>(value1) * sign_extend<int64_t>(value2);
            int64_t round   = static_cast<int64_t>(1) << (shift - 1);
            product = (product + round) >> shift;
            REQUIRE(product >= minimum_s<i32_t> && product <= maximum_s<i32_t>)
            result = product;
        } else {
            result = apply_mul_s(value1, value2);  // low 32-bits of result for i32_t
        }
        tensor_write<out_t>(output, shape, index, result);
    }
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t result = apply_pow<in_out_t>(value1, value2);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

if (in_out_t == shape_t) {
    ERROR_IF(rank(shape) != 0 || rank(shape1) != 0 || rank(shape2) != 0);
    shape_t value1 = tensor_read<shape_t>(input1, [], []);
    shape_t value2 = tensor_read<shape_t>(input2, [], []);
    shape_t result = apply_sub<shape_t>(value1, value2);
    tensor_write<shape_t>(output, [], [], result);
} else {
    ERROR_IF(shape != broadcast_shape(shape1, shape2));
    for_each(index in shape) {
        dim_t index1 = apply_broadcast(shape, shape1, index);
        dim_t index2 = apply_broadcast(shape, shape2, index);
        in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index1);
        in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
        in_out_t result = apply_sub_s<in_out_t>(value1, value2);
        tensor_write<in_out_t>(output, shape, index, result);
    }
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

REQUIRE(length(table) == TABLE_SIZE);
for_each(index in shape) {
    in_t value = tensor_read<in_t>(input, shape, index);
    out_t result;
    if (in_t == i8_t) {
        // value is a signed int, convert to a 0 based index
        result = table[static_cast<int16_t>(value) + 128];
    } else {
        result = apply_lookup_s(static_cast<int16_t>(table), static_cast<int16_t>(value));
    }
    tensor_write<out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    if (is_floating_point(in_out_t) && value1 == -0.0) {
        value1 = 0.0;
    }
    if (static_cast<int32_t>(value1) < 0.0) {
        value1 = apply_sub_s<in_out_t>(0, value1);
    }
    tensor_write<in_out_t>(output, shape, index, value1);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    in_out_t result = ~value1;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    in_out_t result = apply_ceil<in_out_t>(value1);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    in_out_t result = count_leading_zeros(value1);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    in_out_t result = apply_exp<in_out_t>(value1);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    in_out_t result = apply_floor<in_out_t>(value1);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    in_out_t result = apply_log<in_out_t>(value1);
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index);
    in_out_t result = !value1;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(in_out_t != i8_t && input1_zp != 0) // Zero point only for int8_t
ERROR_IF(in_out_t != i8_t && output_zp != 0) // Zero point only for int8_t
for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape, index);
    acc_t value = apply_sub_s<acc_t>(sign_extend<acc_t>(value1),
                                     sign_extend<acc_t>(input1_zp));
    value = apply_sub_s<acc_t>(0, value);
    value = apply_add_s<acc_t>(value, sign_extend<acc_t>(output_zp));
    in_out_t result = truncate<in_out_t>(apply_clip_s<acc_t>(value,
                                                             minimum_s<in_out_t>,
                                                             maximum_s<in_out_t>));
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index);
    in_out_t result = 1.0 / value1;
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

for_each(index in shape) {
    in_out_t value1 = tensor_read<in_out_t>(input1, shape1, index);
    in_out_t result;
    if (value1 < 0) {
        result = NaN;
    }
    else {
        result = 1.0 / apply_sqrt<in_out_t>(value1);
    }
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(broadcast_shape(shape1, shape2), shape3));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    dim_t index3 = apply_broadcast(shape, shape3, index);
    bool_t value1 = tensor_read<bool_t>(input1, shape1, index1);
    in_out_t value2 = tensor_read<in_out_t>(input2, shape2, index2);
    in_out_t value3 = tensor_read<in_out_t>(input3, shape3, index3);
    in_out_t result;
    if (value1) {
         result = value2;
    } else {
         result = value3;
    }
    tensor_write<in_out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_t value1 = tensor_read<in_t>(input1, shape1, index1);
    in_t value2 = tensor_read<in_t>(input2, shape2, index2);
    out_t result;
    if (isNaN(value1) || isNaN(value2))
        result = False;
    else
        result = (value1 == value2) ? True : False;
    tensor_write<out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_t value1 = tensor_read<in_t>(input1, shape1, index1);
    in_t value2 = tensor_read<in_t>(input2, shape2, index2);
    out_t result;
    if (isNaN(value1) || isNaN(value2))
        result = False;
    else
        result = (value1 > value2) ? True : False;
    tensor_write<out_t>(output, shape, index, result);
}

LEVEL_CHECK(rank(shape) <= MAX_RANK);

ERROR_IF(shape != broadcast_shape(shape1, shape2));
for_each(index in shape) {
    dim_t index1 = apply_broadcast(shape, shape1, index);
    dim_t index2 = apply_broadcast(shape, shape2, index);
    in_t value1 = tensor_read<in_t>(input1, shape1, index1);
    in_t value2 = tensor_read<in_t>(input2, shape2, index2);
    out_t result;
    if (isNaN(value1) || isNaN(value2))
        result = False;
    else
        result = (value1 >= value2) ? True : False;
    tensor_write<out_t>(output, shape, index, result);
}

ERROR_IF(axis < 0  || axis >= rank(shape1));
ERROR_IF(shape[axis] != 1);

// Initialize output state to true
for_each(index in shape) {
    tensor_write<in_out_t>(output, shape, index, true);
}
for_each(index in shape1) {
    dim_t out_index = index;
    out_index[axis] = 0;
    in_out_t value = tensor_read<in_out_t>(input, shape1, index);
    in_out_t state = tensor_read<in_out_t>(output, shape, out_index);
    state      = state && value;
    tensor_write<in_out_t>(output, shape, out_index, state);
}

ERROR_IF(axis < 0  || axis >= rank(shape1));
ERROR_IF(shape[axis] != 1);

// Initialize output state to false
for_each(index in shape) {
    tensor_write<in_out_t>(output, shape, index, false);
}
for_each(index in shape1) {
    dim_t out_index = index;
    out_index[axis] = 0;
    in_out_t value = tensor_read<in_out_t>(input, shape1, index);
    in_out_t state = tensor_read<in_out_t>(output, shape, out_index);
    state      = state || value;
    tensor_write<in_out_t>(output, shape, out_index, state);
}