lanes 135 apps/linear_algebra/src/blas_l1_generators.cpp RDom lanes(0, vec_size); lanes 138 apps/linear_algebra/src/blas_l1_generators.cpp result(0) = sum(dot(lanes)); lanes 202 apps/linear_algebra/src/blas_l1_generators.cpp RDom lanes(0, vec_size); lanes 205 apps/linear_algebra/src/blas_l1_generators.cpp result(0) = sum(norm(lanes)); lanes 63 apps/linear_algebra/src/blas_l2_generators.cpp RDom lanes(0, vec_size); lanes 65 apps/linear_algebra/src/blas_l2_generators.cpp sum_lanes(i) += accum_vecs_transpose(i, lanes); lanes 93 apps/linear_algebra/src/blas_l2_generators.cpp .compute_at(result, i).update().unroll(lanes); lanes 40 python_bindings/python/Type.cpp Halide::Type make_handle(int lanes) { lanes 41 python_bindings/python/Type.cpp return Halide::Handle(lanes, nullptr); lanes 47 python_bindings/python/Type.cpp return boost::str(message_format % type_code_to_string(t) % t.bits() % t.lanes()); lanes 67 python_bindings/python/Type.cpp .def("lanes", &Type::lanes, lanes 232 src/AddImageChecks.cpp type_lanes, make_const(UInt(16), type.lanes())}, lanes 237 src/AddImageChecks.cpp (type_lanes == type.lanes()), error)); lanes 38 src/AlignLoads.cpp return mutate(Load::make(load->type.with_lanes(index.type().lanes()), load->name, lanes 39 src/AlignLoads.cpp index, load->image, load->param, const_true(index.type().lanes()))); lanes 70 src/AlignLoads.cpp int lanes = ramp->lanes; lanes 109 src/AlignLoads.cpp Expr dense_index = Ramp::make(dense_base, 1, lanes*stride); lanes 113 src/AlignLoads.cpp expr = Shuffle::make_slice(dense, shift, stride, lanes); lanes 119 src/AlignLoads.cpp if (lanes < native_lanes) { lanes 125 src/AlignLoads.cpp expr = Shuffle::make_slice(native_load, 0, 1, lanes); lanes 129 src/AlignLoads.cpp if (lanes > native_lanes) { lanes 133 src/AlignLoads.cpp for (int i = 0; i < lanes; i += native_lanes) { lanes 134 src/AlignLoads.cpp int slice_lanes = std::min(native_lanes, lanes - i); lanes 147 src/AlignLoads.cpp Expr aligned_load = make_load(op, Ramp::make(aligned_base, 1, lanes*2)); lanes 149 src/AlignLoads.cpp expr = Shuffle::make_slice(aligned_load, aligned_offset, 1, lanes); lanes 568 src/Bounds.cpp make_const(var.type(), op->lanes-1))); lanes 48 src/CodeGen_ARM.cpp int intrin_lanes = t.lanes(); lanes 54 src/CodeGen_ARM.cpp if (t.bits() * t.lanes() == 128) { lanes 60 src/CodeGen_ARM.cpp Type ws = Int(t.bits()*2, t.lanes()); lanes 271 src/CodeGen_ARM.cpp internal_assert(matches[i].type().lanes() == t.lanes()); lanes 323 src/CodeGen_ARM.cpp Type unsigned_type = UInt(t.bits() * 2, t.lanes()); lanes 463 src/CodeGen_ARM.cpp if (op->type.lanes() > 1) { lanes 464 src/CodeGen_ARM.cpp a = ConstantVector::getSplat(op->type.lanes(), a); lanes 533 src/CodeGen_ARM.cpp if (op->type.is_vector() && patterns[i].t.lanes() * patterns[i].t.bits() == 128) { lanes 552 src/CodeGen_ARM.cpp value = call_intrin(op->type, patterns[i].t.lanes(), intrin, {op->a, op->b}); lanes 606 src/CodeGen_ARM.cpp if (op->type.is_vector() && patterns[i].t.lanes() * patterns[i].t.bits() == 128) { lanes 625 src/CodeGen_ARM.cpp value = call_intrin(op->type, patterns[i].t.lanes(), intrin, {op->a, op->b}); lanes 670 src/CodeGen_ARM.cpp int vec_bits = t.bits() * t.lanes(); lanes 717 src/CodeGen_ARM.cpp << intrin_type.lanes() lanes 727 src/CodeGen_ARM.cpp << intrin_type.lanes() lanes 741 src/CodeGen_ARM.cpp int slices = t.lanes() / intrin_type.lanes(); lanes 744 src/CodeGen_ARM.cpp for (int i = 0; i < t.lanes(); i += intrin_type.lanes()) { lanes 746 src/CodeGen_ARM.cpp Expr slice_ramp = Ramp::make(slice_base, ramp->stride, intrin_type.lanes() * num_vecs); lanes 752 src/CodeGen_ARM.cpp slice_args[j] = slice_vector(slice_args[j], i, intrin_type.lanes()); lanes 792 src/CodeGen_ARM.cpp << 'x' << op->value.type().lanes(); lanes 879 src/CodeGen_ARM.cpp int bit_width = op->type.bits() * op->type.lanes(); lanes 901 src/CodeGen_ARM.cpp for (int i = 0; i < op->type.lanes(); i += intrin_lanes) { lanes 924 src/CodeGen_ARM.cpp << 'x' << op->type.lanes(); lanes 949 src/CodeGen_ARM.cpp Type narrow = UInt(a.type().bits()/2, a.type().lanes()); lanes 955 src/CodeGen_ARM.cpp narrow = Int(narrow.bits(), narrow.lanes()); lanes 961 src/CodeGen_ARM.cpp Expr absd = Call::make(UInt(narrow.bits(), narrow.lanes()), Call::absd, lanes 226 src/CodeGen_C.cpp user_assert(type.lanes() == 1) << "Can't use vector types when compiling to C (yet)\n"; lanes 669 src/CodeGen_C.cpp << "{(halide_type_code_t)(" << (int)t.code() << "), " << t.bits() << ", " << t.lanes() << "}, " lanes 103 src/CodeGen_Hexagon.cpp uses_hvx = uses_hvx || op->lanes > 1; lanes 163 src/CodeGen_Hexagon.cpp for (int i = 1; i < op->type.lanes(); i++) { lanes 166 src/CodeGen_Hexagon.cpp predicate = Broadcast::make(condition, predicate.type().lanes()); lanes 254 src/CodeGen_Hexagon.cpp Type i8v2 = i8v1.with_lanes(i8v1.lanes() * 2); lanes 255 src/CodeGen_Hexagon.cpp Type i16v2 = i16v1.with_lanes(i16v1.lanes() * 2); lanes 256 src/CodeGen_Hexagon.cpp Type i32v2 = i32v1.with_lanes(i32v1.lanes() * 2); lanes 257 src/CodeGen_Hexagon.cpp Type u8v2 = u8v1.with_lanes(u8v1.lanes() * 2); lanes 258 src/CodeGen_Hexagon.cpp Type u16v2 = u16v1.with_lanes(u16v1.lanes() * 2); lanes 259 src/CodeGen_Hexagon.cpp Type u32v2 = u32v1.with_lanes(u32v1.lanes() * 2); lanes 616 src/CodeGen_Hexagon.cpp Type split_type = arg_types.front().with_lanes(arg_types.front().lanes() / 2); lanes 1241 src/CodeGen_Hexagon.cpp if (fn->getReturnType()->getVectorNumElements()*2 <= static_cast<unsigned>(result_type.lanes())) { lanes 1442 src/CodeGen_Hexagon.cpp user_assert(op->type.lanes() % (native_vector_bits() * 2 / op->type.bits()) == 0) lanes 1553 src/CodeGen_Hexagon.cpp if (op->lanes * op->type.bits() <= 32) { lanes 113 src/CodeGen_Internal.cpp if (t.lanes() == 1) { lanes 133 src/CodeGen_Internal.cpp return VectorType::get(element_type, t.lanes()); lanes 286 src/CodeGen_Internal.cpp const_true(op->type.lanes())); lanes 295 src/CodeGen_Internal.cpp vector<Expr> lanes; lanes 297 src/CodeGen_Internal.cpp for (int i = 0; i < op->type.lanes(); i++) { lanes 302 src/CodeGen_Internal.cpp lanes.push_back(Call::make(op->type.element_of(), Call::if_then_else, {pred_i, unpredicated_load, lanes 306 src/CodeGen_Internal.cpp expr = Shuffle::make(lanes, ramp); lanes 323 src/CodeGen_Internal.cpp Stmt unpredicated_store = Store::make(op->name, value, index, op->param, const_true(value.type().lanes())); lanes 333 src/CodeGen_Internal.cpp vector<Stmt> lanes; lanes 334 src/CodeGen_Internal.cpp for (int i = 0; i < predicate.type().lanes(); i++) { lanes 339 src/CodeGen_Internal.cpp lanes.push_back(lane); lanes 341 src/CodeGen_Internal.cpp stmt = Block::make(lanes); lanes 797 src/CodeGen_LLVM.cpp ConstantInt::get(i16_t, buf.type().lanes()) lanes 1503 src/CodeGen_LLVM.cpp return codegen_buffer_pointer(base_address, UInt(64, type.lanes()), index); lanes 1573 src/CodeGen_LLVM.cpp width = next_power_of_two(ramp->lanes * stride); lanes 1618 src/CodeGen_LLVM.cpp codegen(reinterpret(op->type, Load::make(UInt(64, op->type.lanes()), op->name, lanes 1644 src/CodeGen_LLVM.cpp Expr base_a = ramp->base, base_b = ramp->base + ramp->lanes; lanes 1674 src/CodeGen_LLVM.cpp Expr ramp_a = Ramp::make(base_a, stride_a, ramp->lanes); lanes 1675 src/CodeGen_LLVM.cpp Expr ramp_b = Ramp::make(base_b, stride_b, ramp->lanes); lanes 1682 src/CodeGen_LLVM.cpp vector<int> indices(ramp->lanes); lanes 1683 src/CodeGen_LLVM.cpp for (int i = 0; i < (ramp->lanes + 1)/2; i++) { lanes 1686 src/CodeGen_LLVM.cpp for (int i = (ramp->lanes + 1)/2; i < ramp->lanes; i++) { lanes 1693 src/CodeGen_LLVM.cpp Expr flipped_base = ramp->base - ramp->lanes + 1; lanes 1695 src/CodeGen_LLVM.cpp Expr flipped_index = Ramp::make(flipped_base, flipped_stride, ramp->lanes); lanes 1700 src/CodeGen_LLVM.cpp vector<int> indices(ramp->lanes); lanes 1701 src/CodeGen_LLVM.cpp for (int i = 0; i < ramp->lanes; i++) { lanes 1702 src/CodeGen_LLVM.cpp indices[i] = ramp->lanes - 1 - i; lanes 1711 src/CodeGen_LLVM.cpp for (int i = 0; i < ramp->lanes; i++) { lanes 1725 src/CodeGen_LLVM.cpp for (int i = 0; i < op->type.lanes(); i++) { lanes 1737 src/CodeGen_LLVM.cpp for (int i = 0; i < op->type.lanes(); i++) { lanes 1756 src/CodeGen_LLVM.cpp Expr broadcast = Broadcast::make(op->base, op->lanes); lanes 1757 src/CodeGen_LLVM.cpp Expr ramp = Ramp::make(make_zero(op->base.type()), op->stride, op->lanes); lanes 1766 src/CodeGen_LLVM.cpp for (int i = 0; i < op->type.lanes(); i++) { lanes 1781 src/CodeGen_LLVM.cpp llvm::Value *CodeGen_LLVM::create_broadcast(llvm::Value *v, int lanes) { lanes 1782 src/CodeGen_LLVM.cpp Constant *undef = UndefValue::get(VectorType::get(v->getType(), lanes)); lanes 1785 src/CodeGen_LLVM.cpp Constant *zeros = ConstantVector::getSplat(lanes, zero); lanes 1790 src/CodeGen_LLVM.cpp value = create_broadcast(codegen(op->value), op->lanes); lanes 1880 src/CodeGen_LLVM.cpp for (int i = 0; i < e.type().lanes(); i++) { lanes 1919 src/CodeGen_LLVM.cpp int store_lanes = value_type.lanes(); lanes 1941 src/CodeGen_LLVM.cpp for (int i = 0; i < op->index.type().lanes(); i++) { lanes 2006 src/CodeGen_LLVM.cpp int load_lanes = load->type.lanes(); lanes 2040 src/CodeGen_LLVM.cpp vector<int> indices(ramp->lanes); lanes 2041 src/CodeGen_LLVM.cpp for (int i = 0; i < ramp->lanes; i++) { lanes 2042 src/CodeGen_LLVM.cpp indices[i] = ramp->lanes - 1 - i; lanes 2050 src/CodeGen_LLVM.cpp Expr flipped_base = ramp->base - ramp->lanes + 1; lanes 2052 src/CodeGen_LLVM.cpp Expr flipped_index = Ramp::make(flipped_base, flipped_stride, ramp->lanes); lanes 2054 src/CodeGen_LLVM.cpp op->param, const_true(op->type.lanes())); lanes 2060 src/CodeGen_LLVM.cpp op->param, const_true(op->type.lanes())); lanes 2125 src/CodeGen_LLVM.cpp llvm::Type *intermediate = llvm_type_of(UInt(32, dst.lanes())); lanes 2140 src/CodeGen_LLVM.cpp llvm::Type *intermediate = llvm_type_of(UInt(32, src.lanes())); lanes 2170 src/CodeGen_LLVM.cpp find_vector_runtime_function(name, op->type.lanes()).first; lanes 2200 src/CodeGen_LLVM.cpp find_vector_runtime_function(name, op->type.lanes()).first; lanes 2392 src/CodeGen_LLVM.cpp internal_assert(t.lanes() == 1); lanes 2746 src/CodeGen_LLVM.cpp t = VectorType::get(t, halide_arg.type().lanes()); lanes 2770 src/CodeGen_LLVM.cpp find_vector_runtime_function(name, op->type.lanes()); lanes 2782 src/CodeGen_LLVM.cpp for (int i = 0; i < op->type.lanes(); i++) { lanes 3079 src/CodeGen_LLVM.cpp Expr v = reinterpret(UInt(64, op->value.type().lanes()), op->value); lanes 3129 src/CodeGen_LLVM.cpp int store_lanes = value_type.lanes(); lanes 3149 src/CodeGen_LLVM.cpp for (int i = 0; i < ramp->lanes; i++) { lanes 3171 src/CodeGen_LLVM.cpp for (int i = 0; i < value_type.lanes(); i++) { lanes 3457 src/CodeGen_LLVM.cpp std::pair<llvm::Function *, int> CodeGen_LLVM::find_vector_runtime_function(const std::string &name, int lanes) { lanes 3467 src/CodeGen_LLVM.cpp while (l < lanes) l *= 2; lanes 279 src/CodeGen_LLVM.h llvm::Value *create_broadcast(llvm::Value *, int lanes); lanes 449 src/CodeGen_LLVM.h std::pair<llvm::Function *, int> find_vector_runtime_function(const std::string &name, int lanes); lanes 27 src/CodeGen_Metal_Dev.cpp if (storage && type.lanes() != 1) { lanes 63 src/CodeGen_Metal_Dev.cpp if (type.lanes() != 1) { lanes 64 src/CodeGen_Metal_Dev.cpp switch (type.lanes()) { lanes 70 src/CodeGen_Metal_Dev.cpp oss << type.lanes(); lanes 178 src/CodeGen_Metal_Dev.cpp << print_type(op->type.with_lanes(op->lanes)) << "(0"; lanes 180 src/CodeGen_Metal_Dev.cpp for (int i = 1; i < op->lanes; ++i) { lanes 184 src/CodeGen_Metal_Dev.cpp print_assignment(op->type.with_lanes(op->lanes), rhs.str()); lanes 191 src/CodeGen_Metal_Dev.cpp rhs << print_type(op->type.with_lanes(op->lanes)) << "(" << id_value << ")"; lanes 193 src/CodeGen_Metal_Dev.cpp print_assignment(op->type.with_lanes(op->lanes), rhs.str()); lanes 272 src/CodeGen_Metal_Dev.cpp for (int i = 0; i < op->type.lanes(); ++i) { lanes 308 src/CodeGen_Metal_Dev.cpp for (int i = 0; i < t.lanes(); ++i) { lanes 40 src/CodeGen_OpenCL_Dev.cpp internal_assert(type.lanes() == 1) << "Encountered vector of bool\n"; lanes 59 src/CodeGen_OpenCL_Dev.cpp if (type.lanes() != 1) { lanes 60 src/CodeGen_OpenCL_Dev.cpp switch (type.lanes()) { lanes 66 src/CodeGen_OpenCL_Dev.cpp oss << type.lanes(); lanes 135 src/CodeGen_OpenCL_Dev.cpp << print_type(op->type.with_lanes(op->lanes)) << ")(0"; lanes 137 src/CodeGen_OpenCL_Dev.cpp for (int i = 1; i < op->lanes; ++i) { lanes 141 src/CodeGen_OpenCL_Dev.cpp print_assignment(op->type.with_lanes(op->lanes), rhs.str()); lanes 147 src/CodeGen_OpenCL_Dev.cpp print_assignment(op->type.with_lanes(op->lanes), id_value); lanes 232 src/CodeGen_OpenCL_Dev.cpp rhs << "vload" << op->type.lanes() lanes 274 src/CodeGen_OpenCL_Dev.cpp for (int i = 0; i < op->type.lanes(); ++i) { lanes 301 src/CodeGen_OpenCL_Dev.cpp stream << "vstore" << t.lanes() << "(" lanes 314 src/CodeGen_OpenCL_Dev.cpp for (int i = 0; i < t.lanes(); ++i) { lanes 448 src/CodeGen_OpenCL_Dev.cpp int op_lanes = op->type.lanes(); lanes 450 src/CodeGen_OpenCL_Dev.cpp int arg_lanes = op->vectors[0].type().lanes(); lanes 458 src/CodeGen_OpenCL_Dev.cpp internal_assert(op->vectors[1].type().lanes() == arg_lanes); lanes 472 src/CodeGen_OpenCL_Dev.cpp int dest_lanes = op->type.lanes(); lanes 477 src/CodeGen_OpenCL_Dev.cpp internal_assert(op->vectors[i].type().lanes() == arg_lanes); lanes 44 src/CodeGen_OpenGLCompute_Dev.cpp user_assert(type.lanes() <= 4) lanes 49 src/CodeGen_OpenGLCompute_Dev.cpp result = map_type(scalar_type).with_lanes(type.lanes()); lanes 58 src/CodeGen_OpenGLCompute_Dev.cpp string s = mapped_type.is_scalar() ? "uint": "uvec" + std::to_string(mapped_type.lanes()); lanes 163 src/CodeGen_OpenGLCompute_Dev.cpp if (op->lanes > 4) lanes 164 src/CodeGen_OpenGLCompute_Dev.cpp internal_error << "GLSL: ramp lanes " << op->lanes << " is not supported\n"; lanes 168 src/CodeGen_OpenGLCompute_Dev.cpp for (int i = 1; i < op->lanes; ++i) { lanes 179 src/CodeGen_OpenGLCompute_Dev.cpp oss << print_type(op->type.with_lanes(op->lanes)) << "(" << id_value << ")"; lanes 180 src/CodeGen_OpenGLCompute_Dev.cpp print_assignment(op->type.with_lanes(op->lanes), oss.str()); lanes 55 src/CodeGen_OpenGL_Dev.cpp user_assert(type.lanes() <= 4) lanes 60 src/CodeGen_OpenGL_Dev.cpp result = map_type(scalar_type).with_lanes(type.lanes()); lanes 74 src/CodeGen_OpenGL_Dev.cpp Type float_type = Float(32, result_type.lanes()); lanes 180 src/CodeGen_OpenGL_Dev.cpp Type float_type = Float(32, op->type.lanes()); lanes 230 src/CodeGen_OpenGL_Dev.cpp oss << "vec" << type.lanes(); lanes 297 src/CodeGen_OpenGL_Dev.cpp int shuffle_lanes = op->type.lanes(); lanes 412 src/CodeGen_OpenGL_Dev.cpp const int lanes = op->type.lanes(); lanes 413 src/CodeGen_OpenGL_Dev.cpp vector<Expr> result(lanes); lanes 414 src/CodeGen_OpenGL_Dev.cpp for (int i = 0; i < lanes; i++) { lanes 459 src/CodeGen_OpenGL_Dev.cpp internal_assert(op->condition.type().lanes() == op->type.lanes()); lanes 460 src/CodeGen_OpenGL_Dev.cpp int lanes = op->type.lanes(); lanes 462 src/CodeGen_OpenGL_Dev.cpp vector<string> ids(lanes); lanes 463 src/CodeGen_OpenGL_Dev.cpp for (int i = 0; i < lanes; i++) { lanes 470 src/CodeGen_OpenGL_Dev.cpp for (int i = 0; i < lanes; i++) { lanes 471 src/CodeGen_OpenGL_Dev.cpp stream << ids[i] << ((i < lanes - 1) ? ", " : ");\n"); lanes 484 src/CodeGen_OpenGL_Dev.cpp if (r && is_zero(r->base) && is_one(r->stride) && r->lanes == 4) { lanes 487 src/CodeGen_OpenGL_Dev.cpp } else if (r && is_zero(r->base) && is_one(r->stride) && r->lanes == 3) { lanes 489 src/CodeGen_OpenGL_Dev.cpp } else if (r && is_zero(r->base) && is_one(r->stride) && r->lanes == 2) { lanes 503 src/CodeGen_OpenGL_Dev.cpp int l = e.type().lanes(); lanes 538 src/CodeGen_OpenGL_Dev.cpp for (int i = 0; i < op->type.lanes(); i++) { lanes 569 src/CodeGen_OpenGL_Dev.cpp for (int i = 0; i < op->value.type().lanes(); i++) { lanes 604 src/CodeGen_OpenGL_Dev.cpp (op->type.lanes() >= 1 && op->type.lanes() <= 4)); lanes 623 src/CodeGen_OpenGL_Dev.cpp switch (op->type.lanes()) { lanes 645 src/CodeGen_OpenGL_Dev.cpp for (int i = 0; i < op->type.lanes(); i++) { lanes 715 src/CodeGen_OpenGL_Dev.cpp Type result_type = Float(32, op->type.lanes()); lanes 803 src/CodeGen_OpenGL_Dev.cpp if (op->lanes > 4) lanes 804 src/CodeGen_OpenGL_Dev.cpp internal_error << "GLSL: ramp lanes " << op->lanes << " is not supported\n"; lanes 808 src/CodeGen_OpenGL_Dev.cpp for (int i = 1; i < op->lanes; ++i) { lanes 31 src/CodeGen_Posix.cpp Expr total_size = make_const(UInt(64), type.lanes() * type.bytes()); lanes 168 src/CodeGen_Posix.cpp llvm_type_of(type.is_handle() ? UInt(64, type.lanes()) : type); lanes 116 src/CodeGen_PowerPC.cpp value = call_intrin(op->type, pattern.type.lanes(), pattern.intrin, matches); lanes 45 src/CodeGen_X86.cpp if (!(ma && mb && t.is_int() && t.bits() == 32 && (t.lanes() >= 4))) { lanes 98 src/CodeGen_X86.cpp if (slice_size < t.lanes()) { lanes 104 src/CodeGen_X86.cpp for (int i = 0; i < op->type.lanes(); i += slice_size) { lanes 119 src/CodeGen_X86.cpp value = slice_vector(value, 0, t.lanes()); lanes 133 src/CodeGen_X86.cpp if (slice_size < t.lanes()) { lanes 139 src/CodeGen_X86.cpp for (int i = 0; i < op->type.lanes(); i += slice_size) { lanes 152 src/CodeGen_X86.cpp value = slice_vector(value, 0, t.lanes()); lanes 182 src/CodeGen_X86.cpp if (slice_size < t.lanes()) { lanes 187 src/CodeGen_X86.cpp for (int i = 0; i < t.lanes(); i += slice_size) { lanes 196 src/CodeGen_X86.cpp value = slice_vector(value, 0, t.lanes()); lanes 270 src/CodeGen_X86.cpp if (op->type.lanes() < pattern.min_lanes) { lanes 284 src/CodeGen_X86.cpp value = call_intrin(op->type, pattern.type.lanes(), pattern.intrin, matches); lanes 299 src/CodeGen_X86.cpp Type signed_type = Int(32, op->type.lanes()); lanes 19 src/ConciseCasts.h return cast(Float(64, e.type().lanes()), e); lanes 23 src/ConciseCasts.h return cast(Float(32, e.type().lanes()), e); lanes 27 src/ConciseCasts.h return cast(Int(64, e.type().lanes()), e); lanes 31 src/ConciseCasts.h return cast(Int(32, e.type().lanes()), e); lanes 35 src/ConciseCasts.h return cast(Int(16, e.type().lanes()), e); lanes 39 src/ConciseCasts.h return cast(Int(8, e.type().lanes()), e); lanes 43 src/ConciseCasts.h return cast(UInt(64, e.type().lanes()), e); lanes 47 src/ConciseCasts.h return cast(UInt(32, e.type().lanes()), e); lanes 51 src/ConciseCasts.h return cast(UInt(16, e.type().lanes()), e); lanes 55 src/ConciseCasts.h return cast(UInt(8, e.type().lanes()), e); lanes 59 src/ConciseCasts.h return saturating_cast(Int(8, e.type().lanes()), e); lanes 63 src/ConciseCasts.h return saturating_cast(UInt(8, e.type().lanes()), e); lanes 67 src/ConciseCasts.h return saturating_cast(Int(16, e.type().lanes()), e); lanes 71 src/ConciseCasts.h return saturating_cast(UInt(16, e.type().lanes()), e); lanes 75 src/ConciseCasts.h return saturating_cast(Int(32, e.type().lanes()), e); lanes 79 src/ConciseCasts.h return saturating_cast(UInt(32, e.type().lanes()), e); lanes 83 src/ConciseCasts.h return saturating_cast(Int(64, e.type().lanes()), e); lanes 87 src/ConciseCasts.h return saturating_cast(UInt(64, e.type().lanes()), e); lanes 322 src/Deinterleave.cpp internal_assert(e.type().lanes() % 2 == 0); lanes 326 src/Deinterleave.cpp d.new_lanes = e.type().lanes()/2; lanes 332 src/Deinterleave.cpp internal_assert(e.type().lanes() % 2 == 0); lanes 336 src/Deinterleave.cpp d.new_lanes = (e.type().lanes()+1)/2; lanes 342 src/Deinterleave.cpp internal_assert(e.type().lanes() % 2 == 0); lanes 348 src/Deinterleave.cpp internal_assert(e.type().lanes() % 2 == 0); lanes 354 src/Deinterleave.cpp internal_assert(e.type().lanes() % 3 == 0); lanes 358 src/Deinterleave.cpp d.new_lanes = (e.type().lanes()+2)/3; lanes 367 src/Deinterleave.cpp d.lane_stride = e.type().lanes(); lanes 384 src/Deinterleave.cpp if (e.type().lanes() <= num_lanes) { lanes 437 src/Deinterleave.cpp if (value.type().lanes() % 2 == 0) { lanes 441 src/Deinterleave.cpp if (value.type().lanes() % 3 == 0) { lanes 464 src/Deinterleave.cpp (r->type.lanes() % i) == 0) { lanes 478 src/Deinterleave.cpp (r->type.lanes() % i) == 0) { lanes 584 src/Deinterleave.cpp const int lanes = r0->lanes; lanes 585 src/Deinterleave.cpp const int64_t expected_stores = stride == 1 ? lanes : stride; lanes 617 src/Deinterleave.cpp if (ri->lanes != lanes) goto fail; lanes 632 src/Deinterleave.cpp if (*offs % lanes != 0) goto fail; lanes 645 src/Deinterleave.cpp if (!is_const(ramp->stride, lanes) || ramp->lanes != lanes) goto fail; lanes 678 src/Deinterleave.cpp load_image, load_param, const_true(t.lanes())); lanes 689 src/Deinterleave.cpp t = t.with_lanes(lanes*stores.size()); lanes 690 src/Deinterleave.cpp Expr index = Ramp::make(base, make_one(base.type()), t.lanes()); lanes 750 src/Deinterleave.cpp check(Load::make(ramp.type(), "buf", ramp, Buffer<>(), Parameter(), const_true(ramp.type().lanes())), lanes 751 src/Deinterleave.cpp Load::make(ramp_a.type(), "buf", ramp_a, Buffer<>(), Parameter(), const_true(ramp_a.type().lanes())), lanes 752 src/Deinterleave.cpp Load::make(ramp_b.type(), "buf", ramp_b, Buffer<>(), Parameter(), const_true(ramp_b.type().lanes()))); lanes 32 src/EliminateBoolVectors.cpp if (t.lanes() > 1 && t.bits() != b.type().bits()) { lanes 50 src/EliminateBoolVectors.cpp if (t.lanes() > 1) { lanes 71 src/EliminateBoolVectors.cpp if (ta.lanes() > 1) { lanes 99 src/EliminateBoolVectors.cpp if (a.type().lanes() > 1) { lanes 127 src/EliminateBoolVectors.cpp Type ty = UInt(8, op->value.type().lanes()); lanes 173 src/EliminateBoolVectors.cpp expr = Broadcast::make(Call::make(Int(8), Call::bool_to_mask, {value}, Call::PureIntrinsic), op->lanes); lanes 175 src/EliminateBoolVectors.cpp expr = Broadcast::make(value, op->lanes); lanes 2202 src/Generator.h static inline Type Bool(int lanes = 1) { return Halide::Bool(lanes); } lanes 2203 src/Generator.h static inline Type Float(int bits, int lanes = 1) { return Halide::Float(bits, lanes); } lanes 2204 src/Generator.h static inline Type Int(int bits, int lanes = 1) { return Halide::Int(bits, lanes); } lanes 2205 src/Generator.h static inline Type UInt(int bits, int lanes = 1) { return Halide::UInt(bits, lanes); } lanes 71 src/HexagonOptimize.cpp int lanes; lanes 73 src/HexagonOptimize.cpp Type with_lanes(Type t) { return t.with_lanes(lanes); } lanes 76 src/HexagonOptimize.cpp if (op->type.lanes() != lanes) { lanes 84 src/HexagonOptimize.cpp if (op->type.lanes() != lanes) { lanes 92 src/HexagonOptimize.cpp if (op->type.lanes() != lanes) { lanes 93 src/HexagonOptimize.cpp expr = Broadcast::make(op->value, lanes); lanes 100 src/HexagonOptimize.cpp WithLanes(int lanes) : lanes(lanes) {} lanes 103 src/HexagonOptimize.cpp Expr with_lanes(Expr x, int lanes) { lanes 104 src/HexagonOptimize.cpp return WithLanes(lanes).mutate(x); lanes 377 src/HexagonOptimize.cpp if (ty.lanes() == 1 && x.type().lanes() > 1) { lanes 382 src/HexagonOptimize.cpp if (ty.lanes() != x.type().lanes()) { lanes 467 src/HexagonOptimize.cpp int lanes = op->type.lanes(); lanes 477 src/HexagonOptimize.cpp mpy_count = find_mpy_ops(op, UInt(8, lanes), UInt(8), 4, mpys, rest); lanes 480 src/HexagonOptimize.cpp mpy_count = find_mpy_ops(op, UInt(8, lanes), Int(8), 4, mpys, rest); lanes 517 src/HexagonOptimize.cpp mpy_count = find_mpy_ops(op, UInt(8, lanes), UInt(8, lanes), 4, mpys, rest); lanes 520 src/HexagonOptimize.cpp mpy_count = find_mpy_ops(op, Int(8, lanes), Int(8, lanes), 4, mpys, rest); lanes 555 src/HexagonOptimize.cpp int lanes = op->type.lanes(); lanes 565 src/HexagonOptimize.cpp mpy_count = find_mpy_ops(op, UInt(8, lanes), Int(8), 2, mpys, rest); lanes 569 src/HexagonOptimize.cpp mpy_count = find_mpy_ops(op, Int(16, lanes), Int(8), 2, mpys, rest); lanes 869 src/HexagonOptimize.cpp Expr replacement = with_lanes(i.second, op->type.lanes()); lanes 1234 src/HexagonOptimize.cpp if (i.type().bits() != op->type.bits() || i.type().lanes() != op->type.lanes()) { lanes 1424 src/HexagonOptimize.cpp if ((op->type.lanes()*op->type.bits()) % (native_vector_bits*2) == 0) { lanes 1591 src/HexagonOptimize.cpp index = simplify(cast(UInt(8).with_lanes(op->type.lanes()), index - base)); lanes 10 src/IR.cpp internal_assert(t.lanes() == v.type().lanes()) << "Cast may not change vector widths\n"; lanes 108 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 120 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 132 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 145 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 157 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 170 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 184 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 198 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 209 src/IR.cpp node->type = Bool(a.type().lanes()); lanes 221 src/IR.cpp condition.type().lanes() == true_value.type().lanes()) lanes 235 src/IR.cpp internal_assert(type.lanes() == index.type().lanes()) << "Vector lanes of Load must match vector lanes of index\n"; lanes 236 src/IR.cpp internal_assert(type.lanes() == predicate.type().lanes()) lanes 249 src/IR.cpp Expr Ramp::make(const Expr &base, const Expr &stride, int lanes) { lanes 254 src/IR.cpp internal_assert(lanes > 1) << "Ramp of lanes <= 1\n"; lanes 258 src/IR.cpp node->type = base.type().with_lanes(lanes); lanes 261 src/IR.cpp node->lanes = lanes; lanes 265 src/IR.cpp Expr Broadcast::make(const Expr &value, int lanes) { lanes 268 src/IR.cpp internal_assert(lanes != 1) << "Broadcast of lanes 1\n"; lanes 271 src/IR.cpp node->type = value.type().with_lanes(lanes); lanes 273 src/IR.cpp node->lanes = lanes; lanes 349 src/IR.cpp internal_assert(value.type().lanes() == index.type().lanes()) << "Vector lanes of Store must match vector lanes of index\n"; lanes 350 src/IR.cpp internal_assert(value.type().lanes() == predicate.type().lanes()) lanes 592 src/IR.cpp input_lanes += i.type().lanes(); lanes 612 src/IR.cpp int lanes = vectors.front().type().lanes(); lanes 615 src/IR.cpp internal_assert(i.type().lanes() == lanes) lanes 620 src/IR.cpp for (int i = 0; i < lanes; i++) { lanes 622 src/IR.cpp indices.push_back(j * lanes + i); lanes 639 src/IR.cpp for (int j = 0; j < vectors[i].type().lanes(); j++) { lanes 648 src/IR.cpp if (begin == 0 && size == vector.type().lanes() && stride == 1) { lanes 665 src/IR.cpp int lanes = vectors.front().type().lanes(); lanes 668 src/IR.cpp if (lanes == 1) { lanes 673 src/IR.cpp if (i.type().lanes() != lanes) { lanes 679 src/IR.cpp if (lanes * vectors.size() != indices.size()) { lanes 684 src/IR.cpp for (int j = 0; j < lanes; j++) { lanes 685 src/IR.cpp if (indices[j * (int)vectors.size() + i] != i * lanes + j) { lanes 712 src/IR.cpp input_lanes += i.type().lanes(); lanes 723 src/IR.cpp input_lanes += i.type().lanes(); lanes 224 src/IR.h int lanes; lanes 226 src/IR.h EXPORT static Expr make(const Expr &base, const Expr &stride, int lanes); lanes 236 src/IR.h int lanes; lanes 238 src/IR.h EXPORT static Expr make(const Expr &value, int lanes); lanes 203 src/IREquality.cpp compare_scalar(a.lanes(), b.lanes()); lanes 68 src/IRMatch.cpp bool lanes_matches = (pattern_type.lanes() == 0) || (pattern_type.lanes() == expr_type.lanes()); lanes 204 src/IRMatch.cpp if (result && e && e->lanes == op->lanes) { lanes 132 src/IRMutator.cpp expr = Ramp::make(base, stride, op->lanes); lanes 139 src/IRMutator.cpp else expr = Broadcast::make(value, op->lanes); lanes 257 src/IROperator.cpp return Broadcast::make(make_const(t.element_of(), val), t.lanes()); lanes 331 src/IROperator.cpp return Broadcast::make(v, b->lanes); lanes 388 src/IROperator.cpp a = Broadcast::make(a, b.type().lanes()); lanes 390 src/IROperator.cpp b = Broadcast::make(b, a.type().lanes()); lanes 392 src/IROperator.cpp internal_assert(a.type().lanes() == b.type().lanes()) << "Can't match types of differing widths"; lanes 417 src/IROperator.cpp int lanes = a.type().lanes(); lanes 418 src/IROperator.cpp a = cast(Int(bits, lanes), a); lanes 419 src/IROperator.cpp b = cast(Int(bits, lanes), b); lanes 430 src/IROperator.cpp Type int_type = Int(32, type.lanes()); lanes 511 src/IROperator.cpp Expr k = cast(Int(32, type.lanes()), k_real); lanes 106 src/IROperator.h EXPORT Expr make_bool(bool val, int lanes = 1); lanes 119 src/IROperator.h EXPORT Expr const_true(int lanes = 1); lanes 123 src/IROperator.h EXPORT Expr const_false(int lanes = 1); lanes 227 src/IROperator.h return Internal::Broadcast::make(cast(t.element_of(), a), t.lanes()); lanes 229 src/IROperator.h internal_assert(b->lanes == t.lanes()); lanes 230 src/IROperator.h return Internal::Broadcast::make(cast(t.element_of(), b->value), t.lanes()); lanes 1234 src/IROperator.h Type t = Float(32, x.type().lanes()); lanes 1252 src/IROperator.h Type t = Float(32, x.type().lanes()); lanes 1271 src/IROperator.h Type t = Float(32, x.type().lanes()); lanes 1288 src/IROperator.h Type t = Float(32, x.type().lanes()); lanes 1298 src/IROperator.h Type t = Bool(x.type().lanes()); lanes 1306 src/IROperator.h Type ft = Float(32, x.type().lanes()); lanes 1322 src/IROperator.h int from_bits = e.type().bits() * e.type().lanes(); lanes 1323 src/IROperator.h int to_bits = t.bits() * t.lanes(); lanes 1543 src/IROperator.h user_assert((zero_val.type().is_float() || zero_val.type().lanes() <= 32)) lanes 39 src/IRPrinter.cpp if (type.lanes() > 1) out << 'x' << type.lanes(); lanes 498 src/IRPrinter.cpp stream << ", " << op->lanes << ")"; lanes 502 src/IRPrinter.cpp stream << "x" << op->lanes << "("; lanes 24 src/Lerp.cpp computation_type = UInt(zero_val.type().bits(), zero_val.type().lanes()); lanes 62 src/Lerp.cpp Cast::make(Float(64, typed_weight.type().lanes()), typed_weight)); lanes 143 src/Lerp.cpp Expr zero_expand = Cast::make(UInt(2 * bits, computation_type.lanes()), lanes 145 src/Lerp.cpp Expr one_expand = Cast::make(UInt(2 * bits, one_val.type().lanes()), lanes 155 src/Lerp.cpp result = Cast::make(UInt(bits, computation_type.lanes()), divided); lanes 130 src/LoopCarry.cpp return Ramp::make(i * t.lanes(), 1, t.lanes()); lanes 393 src/LoopCarry.cpp Buffer<>(), Parameter(), const_true(orig_load->type.lanes())); lanes 400 src/LoopCarry.cpp Parameter(), const_true(orig_load->type.lanes())); lanes 408 src/LoopCarry.cpp Parameter(), const_true(orig_load->type.lanes())); lanes 437 src/LoopCarry.cpp const_true(scratch_idx.type().lanes())); lanes 459 src/LoopCarry.cpp (int)c.size() * loads[c.front()][0]->type.lanes(), lanes 139 src/RemoveUndef.cpp expr = Ramp::make(base, stride, op->lanes); lanes 147 src/RemoveUndef.cpp else expr = Broadcast::make(value, op->lanes); lanes 432 src/Simplify.cpp int64_t min_last_lane = min_base + min_stride * (r->lanes - 1); lanes 433 src/Simplify.cpp int64_t max_last_lane = max_base + max_stride * (r->lanes - 1); lanes 537 src/Simplify.cpp expr = mutate(Broadcast::make(Cast::make(op->type.element_of(), broadcast_value->value), broadcast_value->lanes)); lanes 544 src/Simplify.cpp ramp_value->lanes)); lanes 692 src/Simplify.cpp ramp_a->stride + ramp_b->stride, ramp_a->lanes)); lanes 697 src/Simplify.cpp ramp_a->stride, ramp_a->lanes)); lanes 702 src/Simplify.cpp ramp_b->stride, ramp_b->lanes)); lanes 707 src/Simplify.cpp broadcast_a->lanes); lanes 1040 src/Simplify.cpp ramp_a->stride - ramp_b->stride, ramp_a->lanes)); lanes 1044 src/Simplify.cpp ramp_a->stride, ramp_a->lanes)); lanes 1049 src/Simplify.cpp ramp_b->lanes)); lanes 1053 src/Simplify.cpp broadcast_a->lanes); lanes 1511 src/Simplify.cpp expr = Broadcast::make(mutate(broadcast_a->value * broadcast_b->value), broadcast_a->lanes); lanes 1514 src/Simplify.cpp expr = mutate(Ramp::make(ramp_a->base * m, ramp_a->stride * m, ramp_a->lanes)); lanes 1517 src/Simplify.cpp expr = mutate(Ramp::make(m * ramp_b->base, m * ramp_b->stride, ramp_b->lanes)); lanes 1649 src/Simplify.cpp expr = mutate(Broadcast::make(Div::make(broadcast_a->value, broadcast_b->value), broadcast_a->lanes)); lanes 1667 src/Simplify.cpp ramp_a->lanes)); lanes 1675 src/Simplify.cpp div_imp((int64_t)mod_rem.remainder, ib) == div_imp(mod_rem.remainder + (ramp_a->lanes-1)*ia, ib)) { lanes 1677 src/Simplify.cpp expr = mutate(Broadcast::make(ramp_a->base / broadcast_b->value, ramp_a->lanes)); lanes 2029 src/Simplify.cpp expr = mutate(Broadcast::make(Mod::make(broadcast_a->value, broadcast_b->value), broadcast_a->lanes)); lanes 2078 src/Simplify.cpp expr = mutate(Broadcast::make(ramp_a->base % broadcast_b->value, ramp_a->lanes)); lanes 2086 src/Simplify.cpp div_imp((int64_t)mod_rem.remainder, ib) == div_imp(mod_rem.remainder + (ramp_a->lanes-1)*ia, ib)) { lanes 2089 src/Simplify.cpp expr = mutate(Ramp::make(new_base, ramp_a->stride, ramp_a->lanes)); lanes 2101 src/Simplify.cpp expr = mutate(Ramp::make(new_base, ramp_a->stride, ramp_a->lanes) % b); lanes 2201 src/Simplify.cpp expr = mutate(Broadcast::make(Min::make(broadcast_a->value, broadcast_b->value), broadcast_a->lanes)); lanes 2322 src/Simplify.cpp expr = mutate(Min::make(min_a->a, Broadcast::make(Min::make(broadcast_a_b->value, broadcast_b->value), broadcast_b->lanes))); lanes 2585 src/Simplify.cpp expr = mutate(Broadcast::make(Max::make(broadcast_a->value, broadcast_b->value), broadcast_a->lanes)); lanes 2687 src/Simplify.cpp expr = mutate(Max::make(max_a->a, Broadcast::make(Max::make(broadcast_a_b->value, broadcast_b->value), broadcast_b->lanes))); lanes 2877 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 2882 src/Simplify.cpp for (int i = 0; i < delta.type().lanes(); i++) { lanes 2891 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 2894 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 2918 src/Simplify.cpp broadcast->lanes); lanes 2969 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 2973 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3016 src/Simplify.cpp expr = make_bool(ia < ib, op->type.lanes()); lanes 3019 src/Simplify.cpp expr = make_bool(ua < ub, op->type.lanes()); lanes 3023 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3027 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3031 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3034 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3038 src/Simplify.cpp expr = mutate(Broadcast::make(broadcast_a->value < broadcast_b->value, broadcast_a->lanes)); lanes 3045 src/Simplify.cpp expr = mutate(Broadcast::make(bases_lt, ramp_a->lanes)); lanes 3251 src/Simplify.cpp is_one(mutate(delta_ramp->base + delta_ramp->stride*(delta_ramp->lanes - 1) < 0))) { lanes 3252 src/Simplify.cpp expr = const_true(delta_ramp->lanes); lanes 3256 src/Simplify.cpp expr = const_false(delta_ramp->lanes); lanes 3260 src/Simplify.cpp expr = const_true(delta_ramp->lanes); lanes 3263 src/Simplify.cpp is_one(mutate(delta_ramp->base + delta_ramp->stride*(delta_ramp->lanes - 1) >= 0))) { lanes 3264 src/Simplify.cpp expr = const_false(delta_ramp->lanes); lanes 3267 src/Simplify.cpp 0 <= ia * (delta_ramp->lanes - 1) + mod_rem.remainder && lanes 3268 src/Simplify.cpp ia * (delta_ramp->lanes - 1) + mod_rem.remainder < mod_rem.modulus) { lanes 3270 src/Simplify.cpp expr = Broadcast::make(mutate(LT::make(delta_ramp->base / mod_rem.modulus, 0)), delta_ramp->lanes); lanes 3354 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3360 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3364 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3370 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3376 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3384 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3392 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3401 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3409 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3418 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3426 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3435 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3443 src/Simplify.cpp expr = const_false(op->type.lanes()); lanes 3475 src/Simplify.cpp broadcast_a->lanes == broadcast_b->lanes) { lanes 3477 src/Simplify.cpp expr = Broadcast::make(mutate(And::make(broadcast_a->value, broadcast_b->value)), broadcast_a->lanes); lanes 3530 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3536 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3540 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3546 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3552 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3560 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3568 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3576 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3584 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3592 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3600 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3608 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3616 src/Simplify.cpp expr = const_true(op->type.lanes()); lanes 3620 src/Simplify.cpp broadcast_a->lanes == broadcast_b->lanes) { lanes 3622 src/Simplify.cpp expr = Broadcast::make(mutate(Or::make(broadcast_a->value, broadcast_b->value)), broadcast_a->lanes); lanes 3694 src/Simplify.cpp expr = mutate(Broadcast::make(!n->value, n->lanes)); lanes 3731 src/Simplify.cpp expr = Broadcast::make(condition, true_value.type().lanes()); lanes 3739 src/Simplify.cpp expr = Broadcast::make(mutate(!condition), true_value.type().lanes()); lanes 3861 src/Simplify.cpp expr = Broadcast::make(base, op->lanes); lanes 3866 src/Simplify.cpp expr = Ramp::make(base, stride, op->lanes); lanes 3989 src/Simplify.cpp expr = Broadcast::make(load, b_index->lanes); lanes 4351 src/Simplify.cpp shuffled_predicate = const_true(t.lanes()); lanes 4413 src/Simplify.cpp expr = Ramp::make(r->base, mutate(r->stride / terms), r->lanes * terms); lanes 4466 src/Simplify.cpp if (new_vectors[i].type().lanes() == new_vectors[i-1].type().lanes()) { lanes 4472 src/Simplify.cpp Expr check = mutate(b->value - r->stride * new_vectors[i-1].type().lanes()); lanes 4626 src/Simplify.cpp replacement = substitute(new_name, Ramp::make(new_var, ramp->stride, ramp->lanes), replacement); lanes 4630 src/Simplify.cpp replacement = substitute(new_name, Broadcast::make(new_var, broadcast->lanes), replacement); lanes 4834 src/Simplify.cpp Store::make(op->name, value, index, op->param, const_true(value.type().lanes()))); lanes 6398 src/Simplify.cpp int lanes = 4; lanes 6400 src/Simplify.cpp for (int i = 0; i < lanes; i++) { lanes 6404 src/Simplify.cpp check(concat_vectors(loads), Load::make(Float(32, lanes), "buf", ramp(x, 1, lanes), Buffer<>(), Parameter(), const_true(lanes))); lanes 6427 src/Simplify.cpp Expr value = Load::make(index.type(), "f", index, Buffer<>(), Parameter(), const_true(index.type().lanes())); lanes 272 src/StmtToHtml.cpp print_list(symbol("ramp") + "(", {op->base, op->stride, Expr(op->lanes)}, ")"); lanes 278 src/StmtToHtml.cpp stream << symbol("x") << op->lanes << "("; lanes 211 src/StorageFlattening.cpp stmt = Store::make(op->name, value, idx, output_buf, const_true(value.type().lanes())); lanes 220 src/StorageFlattening.cpp const_true(op->type.lanes())); lanes 34 src/Tracing.cpp (int)type.code(), (int)type.bits(), (int)type.lanes(), lanes 29 src/Type.cpp return Internal::Broadcast::make(element_of().max(), lanes()); lanes 53 src/Type.cpp return Internal::Broadcast::make(element_of().min(), lanes()); lanes 103 src/Type.cpp if (lanes() != other.lanes()) return false; lanes 310 src/Type.h Type(halide_type_code_t code, int bits, int lanes, const halide_handle_cplusplus_type *handle_type = nullptr) lanes 311 src/Type.h : type(code, (uint8_t)bits, (uint16_t)lanes), handle_type(handle_type) { lanes 334 src/Type.h int lanes() const { return type.lanes; } lanes 338 src/Type.h return Type(new_code, bits(), lanes(), lanes 344 src/Type.h return Type(code(), new_bits, lanes(), lanes 362 src/Type.h bool is_vector() const {return lanes() != 1;} lanes 366 src/Type.h bool is_scalar() const {return lanes() == 1;} lanes 385 src/Type.h return code() == other.code() && bits() == other.bits() && lanes() == other.lanes() && lanes 391 src/Type.h return code() != other.code() || bits() != other.bits() || lanes() != other.lanes() || lanes 427 src/Type.h inline Type Int(int bits, int lanes = 1) { lanes 428 src/Type.h return Type(Type::Int, bits, lanes); lanes 432 src/Type.h inline Type UInt(int bits, int lanes = 1) { lanes 433 src/Type.h return Type(Type::UInt, bits, lanes); lanes 437 src/Type.h inline Type Float(int bits, int lanes = 1) { lanes 438 src/Type.h return Type(Type::Float, bits, lanes); lanes 442 src/Type.h inline Type Bool(int lanes = 1) { lanes 443 src/Type.h return UInt(1, lanes); lanes 447 src/Type.h inline Type Handle(int lanes = 1, const halide_handle_cplusplus_type *handle_type = nullptr) { lanes 448 src/Type.h return Type(Type::Handle, 64, lanes, handle_type); lanes 321 src/VaryingAttributes.cpp expr = Broadcast::make(a, op->lanes); lanes 547 src/VaryingAttributes.cpp return Float(e.type().bits(), e.type().lanes()); lanes 626 src/VaryingAttributes.cpp expr = Ramp::make(mutated_base, mutated_stride, op->lanes); lanes 954 src/VaryingAttributes.cpp Parameter(), const_true(op->args[1].type().lanes())); lanes 135 src/VectorizeLoops.cpp Expr last_lane_idx = make_const(r->base.type(), r->lanes-1); lanes 167 src/VectorizeLoops.cpp for (int i = 1; i < e.type().lanes(); i++) { lanes 187 src/VectorizeLoops.cpp int lanes; lanes 194 src/VectorizeLoops.cpp return index * lanes + var; lanes 240 src/VectorizeLoops.cpp int lanes; lanes 254 src/VectorizeLoops.cpp return (bit_size == 32) && (lanes >= 4); lanes 261 src/VectorizeLoops.cpp if (pred.type().lanes() == new_pred.type().lanes()) { lanes 278 src/VectorizeLoops.cpp internal_assert(op->predicate.type().lanes() == lanes); lanes 279 src/VectorizeLoops.cpp internal_assert(op->index.type().lanes() == lanes); lanes 284 src/VectorizeLoops.cpp predicate = mutate(Broadcast::make(op->predicate, lanes)); lanes 285 src/VectorizeLoops.cpp index = mutate(Broadcast::make(op->index, lanes)); lanes 309 src/VectorizeLoops.cpp internal_assert(op->predicate.type().lanes() == lanes); lanes 310 src/VectorizeLoops.cpp internal_assert(op->index.type().lanes() == lanes); lanes 311 src/VectorizeLoops.cpp internal_assert(op->value.type().lanes() == lanes); lanes 317 src/VectorizeLoops.cpp predicate = mutate(Broadcast::make(op->predicate, lanes)); lanes 318 src/VectorizeLoops.cpp value = mutate(Broadcast::make(op->value, lanes)); lanes 319 src/VectorizeLoops.cpp index = mutate(Broadcast::make(op->index, lanes)); lanes 343 src/VectorizeLoops.cpp lanes(vpred.type().lanes()), valid(true), vectorized(false) { lanes 344 src/VectorizeLoops.cpp internal_assert(lanes > 1); lanes 377 src/VectorizeLoops.cpp Expr widen(Expr e, int lanes) { lanes 378 src/VectorizeLoops.cpp if (e.type().lanes() == lanes) { lanes 380 src/VectorizeLoops.cpp } else if (e.type().lanes() == 1) { lanes 381 src/VectorizeLoops.cpp return Broadcast::make(e, lanes); lanes 395 src/VectorizeLoops.cpp Type t = op->type.with_lanes(value.type().lanes()); lanes 419 src/VectorizeLoops.cpp int w = std::max(a.type().lanes(), b.type().lanes()); lanes 449 src/VectorizeLoops.cpp int lanes = std::max(true_value.type().lanes(), false_value.type().lanes()); lanes 450 src/VectorizeLoops.cpp lanes = std::max(lanes, condition.type().lanes()); lanes 452 src/VectorizeLoops.cpp true_value = widen(true_value, lanes); lanes 453 src/VectorizeLoops.cpp false_value = widen(false_value, lanes); lanes 465 src/VectorizeLoops.cpp int w = index.type().lanes(); lanes 485 src/VectorizeLoops.cpp max_lanes = std::max(new_arg.type().lanes(), max_lanes); lanes 582 src/VectorizeLoops.cpp for (int i = 0; i < t.lanes(); i++) { lanes 629 src/VectorizeLoops.cpp max_lanes = std::max(new_arg.type().lanes(), max_lanes); lanes 637 src/VectorizeLoops.cpp max_lanes = std::max(new_value.type().lanes(), max_lanes); lanes 662 src/VectorizeLoops.cpp int lanes = std::max(predicate.type().lanes(), std::max(value.type().lanes(), index.type().lanes())); lanes 663 src/VectorizeLoops.cpp stmt = Store::make(op->name, widen(value, lanes), widen(index, lanes), lanes 664 src/VectorizeLoops.cpp op->param, widen(predicate, lanes)); lanes 669 src/VectorizeLoops.cpp if (op->condition.type().lanes() > 1) { lanes 678 src/VectorizeLoops.cpp int lanes = cond.type().lanes(); lanes 686 src/VectorizeLoops.cpp if (lanes > 1) { lanes 814 src/VectorizeLoops.cpp int lanes = replacement.type().lanes(); lanes 817 src/VectorizeLoops.cpp new_extents.push_back(lanes); lanes 843 src/VectorizeLoops.cpp body = RewriteAccessToVectorAlloc(v, op->name, lanes).mutate(body); lanes 845 src/VectorizeLoops.cpp scope.push(v, Ramp::make(0, 1, lanes)); lanes 851 src/VectorizeLoops.cpp body = substitute(v + widening_suffix, Ramp::make(0, 1, lanes), body); lanes 871 src/VectorizeLoops.cpp s = For::make(var, r->base, r->lanes, ForType::Serial, DeviceAPI::None, s); lanes 881 src/VectorizeLoops.cpp int lanes = replacement.type().lanes(); lanes 883 src/VectorizeLoops.cpp for (int i = lanes - 1; i >= 0; --i) { lanes 896 src/VectorizeLoops.cpp if (i == lanes - 1) { lanes 897 src/VectorizeLoops.cpp result = Broadcast::make(e, lanes); lanes 899 src/VectorizeLoops.cpp Expr cond = (replacement == Broadcast::make(i, lanes)); lanes 900 src/VectorizeLoops.cpp result = Select::make(cond, Broadcast::make(e, lanes), result); lanes 912 src/VectorizeLoops.cpp widening_suffix = ".x" + std::to_string(replacement.type().lanes()); lanes 267 src/runtime/HalideRuntime.h HALIDE_ATTRIBUTE_ALIGN(2) uint16_t lanes; lanes 274 src/runtime/HalideRuntime.h HALIDE_ALWAYS_INLINE halide_type_t(halide_type_code_t code, uint8_t bits, uint16_t lanes = 1) lanes 275 src/runtime/HalideRuntime.h : code(code), bits(bits), lanes(lanes) { lanes 280 src/runtime/HalideRuntime.h HALIDE_ALWAYS_INLINE halide_type_t() : code((halide_type_code_t)0), bits(0), lanes(0) {} lanes 286 src/runtime/HalideRuntime.h lanes == other.lanes); lanes 432 src/runtime/HalideRuntime.h return (const char *)value() + type.lanes * type.bytes(); lanes 96 src/runtime/buffer_t.cpp return buf->type.lanes; lanes 113 src/runtime/buffer_t.cpp dst->type.lanes = 1; lanes 267 src/runtime/to_string.cpp if (t->lanes != 1) { lanes 269 src/runtime/to_string.cpp dst = halide_uint64_to_string(dst, end, t->lanes, 1); lanes 27 src/runtime/tracing.cpp uint32_t value_bytes = (uint32_t)(e->type.lanes * e->type.bytes()); lanes 86 src/runtime/tracing.cpp if (e->type.lanes > 1) { lanes 91 src/runtime/tracing.cpp if ((e->type.lanes > 1) && (i % e->type.lanes) == 0) { lanes 99 src/runtime/tracing.cpp if (e->type.lanes > 1) { lanes 106 src/runtime/tracing.cpp if (e->type.lanes > 1) { lanes 111 src/runtime/tracing.cpp for (int i = 0; i < e->type.lanes; i++) { lanes 148 src/runtime/tracing.cpp if (e->type.lanes > 1) { lanes 238 src/runtime/tracing.cpp event.type.lanes = (uint16_t)type_lanes; lanes 242 src/runtime/tracing.cpp if (event.type.lanes > 1) { lanes 243 src/runtime/tracing.cpp dimensions *= event.type.lanes; lanes 60 test/correctness/fuzz_simplify.cpp return Ramp::make(e1, e2, T.lanes()); lanes 63 test/correctness/fuzz_simplify.cpp return Broadcast::make(e1, T.lanes()); lanes 82 test/correctness/fuzz_simplify.cpp if (maybe_scalar && rng() % T.lanes() == 0) { lanes 131 test/correctness/fuzz_simplify.cpp if (T.lanes() != 1) { lanes 133 test/correctness/fuzz_simplify.cpp return Broadcast::make(e1, T.lanes()); lanes 137 test/correctness/fuzz_simplify.cpp if (T.lanes() != 1) { lanes 140 test/correctness/fuzz_simplify.cpp return Ramp::make(e1, e2, T.lanes()); lanes 163 test/correctness/fuzz_simplify.cpp subT = random_type(T.lanes()); lanes 185 test/correctness/fuzz_simplify.cpp for (int j = 0; j < T.lanes(); j++) { lanes 188 test/correctness/fuzz_simplify.cpp if (T.lanes() != 1) { lanes 12 test/correctness/reschedule.cpp if (ev->type.lanes > 1) { lanes 18 test/correctness/schedule_param.cpp assert(is_const(op->lanes, kExpectedVectorWidth)); lanes 19 test/correctness/specialize.cpp if (ev->type.lanes > 1) { lanes 21 test/correctness/specialize.cpp vector_store_lanes = ev->type.lanes; lanes 83 test/correctness/tracing.cpp assert(ev->dimensions <= 4 && ev->type.lanes <= 4); lanes 92 test/correctness/tracing.cpp e.width = ev->type.lanes; lanes 98 test/correctness/tracing.cpp for (int i = 0; i < ev->type.lanes; i++) { lanes 8 test/correctness/tracing_broadcast.cpp for (int i = 0; i < e->type.lanes; ++i) { lanes 124 test/correctness/vector_math.cpp bool test(int lanes) { lanes 130 test/correctness/vector_math.cpp printf("Testing %sx%d\n", string_of_type<A>(), lanes); lanes 147 test/correctness/vector_math.cpp f1.vectorize(x, lanes); lanes 164 test/correctness/vector_math.cpp f2.vectorize(x, lanes); lanes 181 test/correctness/vector_math.cpp f3.vectorize(x, lanes); lanes 198 test/correctness/vector_math.cpp f4.vectorize(x, lanes); lanes 218 test/correctness/vector_math.cpp f5.vectorize(x, lanes); lanes 243 test/correctness/vector_math.cpp f5a.vectorize(y, lanes); lanes 263 test/correctness/vector_math.cpp f6.update().vectorize(x, lanes); lanes 284 test/correctness/vector_math.cpp f7.vectorize(x, lanes); lanes 300 test/correctness/vector_math.cpp f8.vectorize(x, lanes); lanes 318 test/correctness/vector_math.cpp f9.vectorize(x, lanes); lanes 343 test/correctness/vector_math.cpp f10.vectorize(x, lanes); lanes 366 test/correctness/vector_math.cpp f11.vectorize(x, lanes); lanes 383 test/correctness/vector_math.cpp f12.vectorize(x, lanes); lanes 400 test/correctness/vector_math.cpp f13.vectorize(x, lanes); lanes 436 test/correctness/vector_math.cpp f15.vectorize(x, lanes); lanes 437 test/correctness/vector_math.cpp f16.vectorize(x, lanes); lanes 572 test/correctness/vector_math.cpp weight = cast(UInt(t.bits(), t.lanes()), max(0, weight)); lanes 607 test/correctness/vector_math.cpp f22.vectorize(x, lanes); lanes 9 test/correctness/vectorize_guard_with_if.cpp if (e->type.lanes > 1) { lanes 210 util/HalideTraceViz.cpp loads += p.type.lanes; lanes 215 util/HalideTraceViz.cpp stores += p.type.lanes; lanes 219 util/HalideTraceViz.cpp for (int i = 0; i < std::min(16, p.dimensions / p.type.lanes); i++) { lanes 220 util/HalideTraceViz.cpp for (int lane = 0; lane < p.type.lanes; lane++) { lanes 221 util/HalideTraceViz.cpp int coord = p.get_coord(i*p.type.lanes + lane); lanes 232 util/HalideTraceViz.cpp for (int i = 0; i < p.type.lanes; i++) { lanes 774 util/HalideTraceViz.cpp halide_clock += fi.config.store_cost * p.type.lanes; lanes 777 util/HalideTraceViz.cpp halide_clock += fi.config.load_cost * p.type.lanes; lanes 784 util/HalideTraceViz.cpp assert(p.dimensions >= p.type.lanes * fi.config.dims); lanes 785 util/HalideTraceViz.cpp if (p.dimensions >= p.type.lanes * fi.config.dims) { lanes 786 util/HalideTraceViz.cpp for (int lane = 0; lane < p.type.lanes; lane++) { lanes 791 util/HalideTraceViz.cpp int a = p.get_coord(d * p.type.lanes + lane); lanes 827 util/HalideTraceViz.cpp uint32_t channel = p.get_coord(fi.config.color_dim * p.type.lanes + lane);