Branch data Line data Source code
1 : : #pragma once
2 : :
3 : : #include <algorithm>
4 : : #include "backend/WebAssembly.hpp"
5 : : #include <concepts>
6 : : #include <cstdlib>
7 : : #include <deque>
8 : : #include <experimental/type_traits>
9 : : #include <functional>
10 : : #include <iostream>
11 : : #include <list>
12 : : #include <memory>
13 : : #include <mutable/util/concepts.hpp>
14 : : #include <mutable/util/fn.hpp>
15 : : #include <mutable/util/macro.hpp>
16 : : #include <mutable/util/tag.hpp>
17 : : #include <mutable/util/type_traits.hpp>
18 : : #include <numeric>
19 : : #include <tuple>
20 : : #include <type_traits>
21 : : #include <utility>
22 : :
23 : : // Binaryen
24 : : #include <wasm-binary.h>
25 : : #include <wasm-builder.h>
26 : : #include <wasm-interpreter.h>
27 : : #include <wasm-validator.h>
28 : : #include <wasm.h>
29 : :
30 : :
31 : : namespace m {
32 : :
33 : : #if !defined(NDEBUG) && defined(M_ENABLE_SANITY_FIELDS)
34 : : namespace dsl_options {
35 : :
36 : : /** Whether there must not be any ternary logic, i.e. NULL value computation. Note that NULL values have different
37 : : * origins, e.g. NULL values stored in a table or default aggregate values in an aggregation operator. */
38 : : static bool insist_no_ternary_logic = false;
39 : :
40 : : }
41 : :
42 : : #define M_insist_no_ternary_logic() M_insist(not m::dsl_options::insist_no_ternary_logic, "ternary logic must not occur")
43 : :
44 : : #else
45 : : #define M_insist_no_ternary_logic()
46 : :
47 : : #endif
48 : :
49 : : // forward declarations
50 : : struct Table;
51 : :
52 : : namespace wasm {
53 : :
54 : : /*======================================================================================================================
55 : : * Concepts needed for forward declarations
56 : : *====================================================================================================================*/
57 : :
58 : : /** Check whether \tparam T is a primitive type and not decayable. */
59 : : template<typename T>
60 : : concept dsl_primitive = primitive<T> and not decayable<T>;
61 : :
62 : : /** Check whether \tparam T is a pointer to primitive type and not decayable. */
63 : : template<typename T>
64 : : concept dsl_pointer_to_primitive = pointer_to_primitive<T> and not decayable<T>;
65 : :
66 : :
67 : : /*======================================================================================================================
68 : : * Type forward declarations
69 : : *====================================================================================================================*/
70 : :
71 : : /** Declares the kind of a variable: local, parameter, or global. */
72 : : enum class VariableKind {
73 : : Local,
74 : : Param,
75 : : Global,
76 : : };
77 : :
78 : : struct Allocator; // for use in Module
79 : : struct Bit; // for use in PrimitiveExpr
80 : : template<std::size_t = 1> struct LocalBit; // for use in Module
81 : : struct LocalBitmap; // for use in Module
82 : : struct LocalBitvector; // for use in Module
83 : : template<typename> struct invoke_interpreter; // for unittests only
84 : : template<typename> struct invoke_v8; // for unittests only
85 : : template<typename, std::size_t = 1> struct PrimitiveExpr;
86 : : template<typename, std::size_t = 1> struct Expr;
87 : : template<typename, VariableKind, bool, std::size_t = 1> struct Variable;
88 : : template<typename, std::size_t = 1> struct Parameter;
89 : :
90 : : namespace detail {
91 : :
92 : : template<typename, VariableKind, bool, std::size_t> class variable_storage;
93 : :
94 : : template<dsl_primitive, std::size_t, bool> struct the_reference;
95 : :
96 : : }
97 : :
98 : : template<typename T, std::size_t L = 1>
99 : : using Reference = detail::the_reference<T, L, false>;
100 : : template<typename T, std::size_t L = 1>
101 : : using ConstReference = detail::the_reference<T, L, true>;
102 : :
103 : :
104 : : /*======================================================================================================================
105 : : * Concepts and meta types
106 : : *====================================================================================================================*/
107 : :
108 : : /** Helper type to deduce the `PrimitiveExpr<U>` type given a type \tparam T. */
109 : : template<typename T>
110 : : struct primitive_expr;
111 : :
112 : : /** Specialization for decayable \tparam T. */
113 : : template<decayable T>
114 : : requires requires { typename primitive_expr<std::decay_t<T>>::type; }
115 : : struct primitive_expr<T>
116 : : { using type = typename primitive_expr<std::decay_t<T>>::type; };
117 : :
118 : : /** Specialization for primitive type \tparam T. */
119 : : template<dsl_primitive T>
120 : : struct primitive_expr<T>
121 : : { using type = PrimitiveExpr<T, 1>; };
122 : :
123 : : /** Specialization for pointer to primitive type \tparam T. */
124 : : template<dsl_pointer_to_primitive T>
125 : : struct primitive_expr<T>
126 : : { using type = PrimitiveExpr<T, 1>; };
127 : :
128 : : /** Specialization for \tparam T being `PrimitiveExpr<T, L>` already. */
129 : : template<typename T, std::size_t L>
130 : : struct primitive_expr<PrimitiveExpr<T, L>>
131 : : { using type = PrimitiveExpr<T, L>; };
132 : :
133 : : /** Specialization for \tparam T being a `Variable<T, Kind, false, L>` (i.e. a `Variable` that *cannot* be `NULL`). */
134 : : template<typename T, VariableKind Kind, std::size_t L>
135 : : struct primitive_expr<Variable<T, Kind, false, L>>
136 : : { using type = PrimitiveExpr<T, L>; };
137 : :
138 : : /** Specialization for \tparam T being a `Parameter` (i.e. a local `Variable` that *cannot* be `NULL`). */
139 : : template<typename T, std::size_t L>
140 : : struct primitive_expr<Parameter<T, L>>
141 : : { using type = PrimitiveExpr<T, L>; };
142 : :
143 : : /** Specialization for \tparam T being a `the_reference`. */
144 : : template<typename T, std::size_t L, bool IsConst>
145 : : struct primitive_expr<detail::the_reference<T, L, IsConst>>
146 : : { using type = PrimitiveExpr<T, L>; };
147 : :
148 : : /** Convenience alias for `primitive_expr`. */
149 : : template<typename T>
150 : : using primitive_expr_t = typename primitive_expr<T>::type;
151 : :
152 : :
153 : : /** Detects whether a type \tparam T is convertible to `PrimitiveExpr<U>`. */
154 : : template<typename T>
155 : : concept primitive_convertible = not pointer<T> and requires { typename primitive_expr_t<T>; };
156 : :
157 : :
158 : : /** Helper type to deduce the `Expr<U>` type given a \tparam T. */
159 : : template<typename T>
160 : : struct expr;
161 : :
162 : : /** Specialization for decayable \tparam T. */
163 : : template<decayable T>
164 : : requires requires { typename expr<std::decay_t<T>>::type; }
165 : : struct expr<T>
166 : : { using type = typename expr<std::decay_t<T>>::type; };
167 : :
168 : : /** Specialization for primitive type \tparam T. */
169 : : template<dsl_primitive T>
170 : : struct expr<T>
171 : : { using type = Expr<T, 1>; };
172 : :
173 : : /** Specialization for pointer to primitive type \tparam T. */
174 : : template<dsl_pointer_to_primitive T>
175 : : struct expr<T>
176 : : { using type = Expr<T, 1>; };
177 : :
178 : : /** Specialization for \tparam T being a `PrimitiveExpr<T, L>`. */
179 : : template<typename T, std::size_t L>
180 : : struct expr<PrimitiveExpr<T, L>>
181 : : { using type = Expr<T, L>; };
182 : :
183 : : /** Specialization for \tparam T being `Expr<T, L>` already. */
184 : : template<typename T, std::size_t L>
185 : : struct expr<Expr<T, L>>
186 : : { using type = Expr<T, L>; };
187 : :
188 : : /** Specialization for \tparam T being a `Variable<T, Kind, CanBeNull, L>`. */
189 : : template<typename T, VariableKind Kind, bool CanBeNull, std::size_t L>
190 : : struct expr<Variable<T, Kind, CanBeNull, L>>
191 : : { using type = Expr<T, L>; };
192 : :
193 : : /** Specialization for \tparam T being a `Parameter` (i.e. a local `Variable` that *cannot* be `NULL`). */
194 : : template<typename T, std::size_t L>
195 : : struct expr<Parameter<T, L>>
196 : : { using type = Expr<T, L>; };
197 : :
198 : : /** Specialization for \tparam T being a `the_reference`. */
199 : : template<typename T, std::size_t L, bool IsConst>
200 : : struct expr<detail::the_reference<T, L, IsConst>>
201 : : { using type = Expr<T, L>; };
202 : :
203 : : /** Convenience alias for `expr`. */
204 : : template<typename T>
205 : : using expr_t = typename expr<T>::type;
206 : :
207 : :
208 : : /** Detect whether a type \tparam T is convertible to `Expr<U>`. */
209 : : template<typename T>
210 : : concept expr_convertible = not pointer<T> and requires { typename expr_t<T>; };
211 : :
212 : :
213 : : namespace detail {
214 : :
215 : : /** Converts a compile-time type into a runtime-type `::wasm::Type`. */
216 : : template<typename, std::size_t>
217 : : struct wasm_type_helper;
218 : :
219 : : /** Specialization for \tparam T being void. */
220 : : template<typename T>
221 : : requires (std::is_void_v<T>)
222 : : struct wasm_type_helper<T, 1>
223 : : {
224 : 7545 : ::wasm::Type operator()() const { return ::wasm::Type(::wasm::Type::none); }
225 : : };
226 : :
227 : : /** Specialization for \tparam T being integral. */
228 : : template<std::integral T>
229 : : struct wasm_type_helper<T, 1>
230 : : {
231 : 29777 : ::wasm::Type operator()() const {
232 : : /* NOTE: there are no unsigned types, only unsigned operations */
233 : : if constexpr (sizeof(T) <= 4)
234 : 23120 : return ::wasm::Type(::wasm::Type::i32);
235 : : if constexpr (sizeof(T) == 8)
236 : 6657 : return ::wasm::Type(::wasm::Type::i64);
237 : : M_unreachable("illegal type");
238 : : };
239 : : };
240 : :
241 : : /** Specialization for \tparam T being floating point. */
242 : : template<std::floating_point T>
243 : : struct wasm_type_helper<T, 1>
244 : : {
245 : 736 : ::wasm::Type operator()()
246 : : {
247 : : if constexpr (sizeof(T) <= 4)
248 : 102 : return ::wasm::Type(::wasm::Type::f32);
249 : : if constexpr (sizeof(T) == 8)
250 : 634 : return ::wasm::Type(::wasm::Type::f64);
251 : : M_unreachable("illegal type");
252 : : };
253 : : };
254 : :
255 : : /** Specialization for \tparam T being pointer to primitive. */
256 : : template<dsl_pointer_to_primitive T, std::size_t L>
257 : : struct wasm_type_helper<T, L>
258 : : {
259 : 1381 : ::wasm::Type operator()() { return ::wasm::Type(::wasm::Type::i32); };
260 : : };
261 : :
262 : : /** Specialization for \tparam T being vectorial. */
263 : : template<dsl_primitive T, std::size_t L>
264 : : requires (L > 1)
265 : : struct wasm_type_helper<T, L>
266 : : {
267 : 852 : ::wasm::Type operator()() const { return ::wasm::Type(::wasm::Type::v128); }
268 : : };
269 : :
270 : : /** Specialization for \tparam T being a `PrimitiveExpr`. E.g. used for `Module::emit_function_import()` to enable
271 : : * imports of functions with vectorial return or parameter types. */
272 : : template<typename T, std::size_t L>
273 : : struct wasm_type_helper<PrimitiveExpr<T, L>, 1>
274 : : {
275 : 3547 : ::wasm::Type operator()() { return wasm_type_helper<T, L>{}(); };
276 : : };
277 : :
278 : : /** Specialization for \tparam T being function with parameters. */
279 : : template<typename ReturnType, typename... ParamTypes>
280 : : struct wasm_type_helper<ReturnType(ParamTypes...), 1>
281 : : {
282 : 6981 : ::wasm::Signature operator()()
283 : : {
284 : 6981 : return ::wasm::Signature(
285 : 6981 : /* params= */ { wasm_type_helper<ParamTypes, 1>{}()... },
286 : 6981 : /* result= */ wasm_type_helper<ReturnType, 1>{}());
287 : : };
288 : : };
289 : :
290 : : }
291 : :
292 : : template<typename T, std::size_t L>
293 : 32952 : auto wasm_type() { return detail::wasm_type_helper<T, L>{}(); }
294 : :
295 : :
296 : : /** Helper type to deduce the signed integral type with a given byte width \tparam W. */
297 : : template<std::size_t W>
298 : : struct _int;
299 : :
300 : : template<>
301 : : struct _int<1>
302 : : { using type = int8_t; };
303 : :
304 : : template<>
305 : : struct _int<2>
306 : : { using type = int16_t; };
307 : :
308 : : template<>
309 : : struct _int<4>
310 : : { using type = int32_t; };
311 : :
312 : : template<>
313 : : struct _int<8>
314 : : { using type = int64_t; };
315 : :
316 : : template<std::size_t W>
317 : : using int_t = typename _int<W>::type;
318 : :
319 : : /** Helper type to deduce the unsigned integral type with a given byte width \tparam W. */
320 : : template<std::size_t W>
321 : : struct uint;
322 : :
323 : : template<>
324 : : struct uint<1>
325 : : { using type = uint8_t; };
326 : :
327 : : template<>
328 : : struct uint<2>
329 : : { using type = uint16_t; };
330 : :
331 : : template<>
332 : : struct uint<4>
333 : : { using type = uint32_t; };
334 : :
335 : : template<>
336 : : struct uint<8>
337 : : { using type = uint64_t; };
338 : :
339 : : template<std::size_t W>
340 : : using uint_t = typename uint<W>::type;
341 : :
342 : :
343 : : /*======================================================================================================================
344 : : * Wasm_insist(COND [, MSG])
345 : : *
346 : : * Similarly to `M_insist()`, checks a condition in debug build and prints location information and an optional
347 : : * message if it evaluates to `false`. However, the condition is checked at runtime inside the Wasm code.
348 : : *====================================================================================================================*/
349 : :
350 : : #ifndef NDEBUG
351 : :
352 : : #ifdef M_ENABLE_SANITY_FIELDS
353 : : #define WASM_INSIST2_(COND, MSG) ({ \
354 : : auto old = std::exchange(m::dsl_options::insist_no_ternary_logic, false); \
355 : : m::wasm::Module::Get().emit_insist((COND), __FILE__, __LINE__, (MSG)); \
356 : : m::dsl_options::insist_no_ternary_logic = old; \
357 : : })
358 : : #else
359 : : #define WASM_INSIST2_(COND, MSG) ({ \
360 : : m::wasm::Module::Get().emit_insist((COND), __FILE__, __LINE__, (MSG)); \
361 : : })
362 : : #endif
363 : :
364 : : #define WASM_INSIST1_(COND) WASM_INSIST2_((COND), nullptr)
365 : :
366 : : #else
367 : : #define WASM_INSIST2_(COND, MSG) while (0) { ((void) (COND), (void) (MSG)); }
368 : : #define WASM_INSIST1_(COND) while (0) { ((void) (COND)); }
369 : :
370 : : #endif
371 : :
372 : : #define WASM_GET_INSIST_(XXX, _1, _2, NAME, ...) NAME
373 : : #define Wasm_insist(...) WASM_GET_INSIST_(XXX, ##__VA_ARGS__, WASM_INSIST2_, WASM_INSIST1_)(__VA_ARGS__)
374 : :
375 : :
376 : : /*######################################################################################################################
377 : : * TYPE DEFINITIONS
378 : : *####################################################################################################################*/
379 : :
380 : : /*======================================================================================================================
381 : : * Boxing types
382 : : *====================================================================================================================*/
383 : :
384 : : /** Stores the "branch targets" introduced by control flow structures, i.e. loops.
385 : : *
386 : : * The "break" target identifies the parent `::wasm::Block` of the loop to break out of. The "continue" target
387 : : * identifies the `::wasm::Loop` to reiterate. */
388 : : struct branch_target_t
389 : : {
390 : : ///> the break target
391 : : ::wasm::Name brk;
392 : : ///> the continue target
393 : : ::wasm::Name continu;
394 : : ///> the continue condition (may be `nullptr` if there is no condition)
395 : : ::wasm::Expression *condition = nullptr;
396 : :
397 : 288 : branch_target_t(::wasm::Name brk, ::wasm::Name continu, ::wasm::Expression *condition)
398 : 288 : : brk(brk), continu(continu), condition(condition)
399 : 288 : { }
400 : : };
401 : :
402 : :
403 : : /*======================================================================================================================
404 : : * Helper functions
405 : : *====================================================================================================================*/
406 : :
407 : : /** A helper type to print Wasm types. Use the `param_pack_t` helper since `template<typename..., std::size_t...>`
408 : : * would not be allowed. */
409 : : template<param_pack Ts, std::size_t... Ls>
410 : : struct print_types;
411 : :
412 : : /** Prints the Wasm type for \tparam L values of type \tparam T, recurse to print \tparam Ls values of
413 : : * types \tparam Ts. */
414 : : template<typename T, typename... Ts, std::size_t L, std::size_t... Ls>
415 : : requires (sizeof...(Ts) == sizeof...(Ls))
416 : : struct print_types<param_pack_t<T, Ts...>, L, Ls...>
417 : : {
418 : : friend std::ostream & operator<<(std::ostream &out, print_types) {
419 : : return out << wasm_type<T, L>() << ", " << print_types<Ts..., Ls...>{};
420 : : }
421 : : };
422 : :
423 : : /** Prints the Wasm type for \tparam L values of type \tparam T. */
424 : : template<typename T, std::size_t L>
425 : : struct print_types<param_pack_t<T>, L>
426 : : {
427 : : friend std::ostream & operator<<(std::ostream &out, print_types) {
428 : : return out << wasm_type<T, L>();
429 : : }
430 : : };
431 : :
432 : : /** Creates a unique name from a given \p prefix and a \p counter. Increments `counter`. */
433 : 19427 : inline std::string unique(std::string prefix, unsigned &counter)
434 : : {
435 [ + + ]: 19427 : static thread_local std::ostringstream oss;
436 [ + - + - ]: 19427 : oss.str("");
437 : 19427 : oss << prefix << '<' << counter++ << '>';
438 : 19427 : return oss.str();
439 : 0 : }
440 : :
441 : : /** Creates a `::wasm::Literal` of type \tparam T from a given \p value. Used to solve macOS ambiguity. */
442 : : template<typename T, std::size_t L, bool = false, typename U>
443 : : requires (L == 1) and std::floating_point<T> and std::floating_point<U>
444 : 614 : inline ::wasm::Literal make_literal(U value)
445 : : {
446 : 614 : return ::wasm::Literal(T(value));
447 : : }
448 : :
449 : : /** Creates a `::wasm::Literal` of type \tparam T from a given \p value. Used to solve macOS ambiguity. */
450 : : template<typename T, std::size_t L, bool = false, typename U>
451 : : requires (L == 1) and signed_integral<T> and integral<U>
452 : 4580 : inline ::wasm::Literal make_literal(U value)
453 : : {
454 [ + + + - : 4580 : return sizeof(T) <= 4 ? ::wasm::Literal(int32_t(value))
# # + - -
+ + - + -
# # + - +
+ + - + -
# # + - -
+ # # ]
455 : 432 : : ::wasm::Literal(int64_t(value));
456 : : }
457 : :
458 : : /** Creates a `::wasm::Literal` of type \tparam T from a given \p value. Used to solve macOS ambiguity. */
459 : : template<typename T, std::size_t L, bool = false, typename U>
460 : : requires (L == 1) and unsigned_integral<T> and integral<U>
461 : 16144 : inline ::wasm::Literal make_literal(U value)
462 : : {
463 [ + + + - : 16144 : return sizeof(T) <= 4 ? ::wasm::Literal(uint32_t(value))
+ - + - +
- + + + +
+ - + - +
- - + # #
# # # # #
# + + # #
+ - # # #
# ]
464 : 5046 : : ::wasm::Literal(uint64_t(value));
465 : : }
466 : :
467 : : /** Creates a `::wasm::Literal` of type \tparam T from a given \p value. Used to solve macOS ambiguity. */
468 : : template<typename T, std::size_t L, bool VecRepr = false, typename U>
469 : : requires (L == 1) and boolean<T> and boolean<U>
470 : 1564 : inline ::wasm::Literal make_literal(U value)
471 : : {
472 : 3128 : return M_CONSTEXPR_COND(VecRepr, ::wasm::Literal(0U - uint32_t(value)), ::wasm::Literal(uint32_t(value)));
473 : : }
474 : :
475 : : /** Creates a `::wasm::Literal` of type \tparam T from a given \p value. Used to solve macOS ambiguity. */
476 : : template<typename T, std::size_t L, bool = false>
477 : : requires (L == 1) and std::is_pointer_v<T>
478 : 0 : inline ::wasm::Literal make_literal(uint32_t value)
479 : : {
480 : 0 : return ::wasm::Literal(uint32_t(value));
481 : : }
482 : :
483 : : /** Creates a `::wasm::Literal` for \tparam L values of type \tparam T from a given \p value. Used to solve macOS
484 : : * ambiguity. */
485 : : template<typename T, std::size_t L, bool = false, typename U>
486 : : requires (L > 1) and (L * sizeof(T) <= 16) and
487 : : requires (U value) { make_literal<T, 1>(value); }
488 : 294 : inline ::wasm::Literal make_literal(U value)
489 : : {
490 : 294 : std::array<::wasm::Literal, 16 / sizeof(T)> vec; // must be fully utilized vector
491 [ + - + - : 294 : auto it = std::fill_n(vec.begin(), L, make_literal<T, 1, true>(value)); // fill range [0, L) with value
+ - + - +
- + - # #
# # # # #
# + - + -
+ - + - +
- + - # #
# # # # #
# + - + -
# # # # +
- + - # #
# # + - +
- # # # #
# # # # #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
492 [ + - + - : 294 : std::fill(it, vec.end(), make_literal<T, 1>(T(0))); // fill range [L, end) with 0
+ - + - +
- + - # #
# # # # #
# + - + -
+ - + - +
- + - # #
# # # # #
# + - + -
# # # # +
- + - # #
# # + - +
- # # # #
# # # # #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
493 [ + - + - : 294 : return ::wasm::Literal(vec);
+ - # # #
# + - + -
+ - # # #
# + - # #
+ - # # +
- # # # #
# # + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
494 : 294 : }
495 : :
496 : : /** Creates a `::wasm::Literal` for \tparam L values of type \tparam T from a given \p value. Used to solve macOS
497 : : * ambiguity. */
498 : : template<typename T, std::size_t L, bool = false, typename U>
499 : : requires (L > 1) and (L * sizeof(T) > 16) and ((L * sizeof(T)) % 16 == 0) and
500 : : requires (U value) { make_literal<T, 1>(value); }
501 : 30 : inline std::array<::wasm::Literal, (L * sizeof(T)) / 16> make_literal(U value)
502 : : {
503 : 30 : std::array<::wasm::Literal, 16 / sizeof(T)> vec;
504 [ + - + - : 30 : vec.fill(make_literal<T, 1, true>(value)); // fill single fully utilized vector with value
# # # # #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
505 [ + - # # : 30 : ::wasm::Literal elem(vec);
# # + - #
# # # # #
# # # # #
# # # + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # ]
506 : :
507 [ + - # # : 30 : std::array<::wasm::Literal, (L * sizeof(T)) / 16> literals;
# # + - #
# # # # #
# # # # #
# # # + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # ]
508 [ + - # # : 30 : literals.fill(elem); // fill each vector with single vectorial literal
# # + - #
# # # # #
# # # # #
# # # + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # ]
509 : 30 : return literals;
510 [ + - # # : 30 : }
# # + - #
# # # # #
# # # # #
# # # + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # ]
511 : :
512 : : /** Creates a `::wasm::Literal` for \tparam L values of type \tparam T from given \p values. Used to solve macOS
513 : : * ambiguity. */
514 : : template<typename T, std::size_t L, bool = false, typename... Us>
515 : : requires (L > 1) and (L * sizeof(T) <= 16) and (L == sizeof...(Us)) and
516 : : requires (Us... values) { (make_literal<T, 1>(values), ...); }
517 : 368 : inline ::wasm::Literal make_literal(Us... values)
518 : : {
519 : 368 : std::array<::wasm::Literal, 16 / sizeof(T)> vec; // must be fully utilized vector
520 : 368 : auto it = vec.begin();
521 [ + - + - : 368 : ((*(it++) = make_literal<T, 1, true>(values)), ...); // fill range [0, L) with values
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
522 [ + - + - : 368 : std::fill(it, vec.end(), make_literal<T, 1>(T(0))); // fill range [L, end) with 0
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
523 [ + - + - : 368 : return ::wasm::Literal(vec);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
524 : 368 : }
525 : :
526 : : /** Creates a `::wasm::Literal` for \tparam L values of type \tparam T from given \p values. Used to solve macOS
527 : : * ambiguity. */
528 : : template<typename T, std::size_t L, bool = false, typename... Us>
529 : : requires (L > 1) and (L * sizeof(T) > 16) and ((L * sizeof(T)) % 16 == 0) and (L == sizeof...(Us)) and
530 : : requires (Us... values) { (make_literal<T, 1>(values), ...); }
531 : 20 : inline std::array<::wasm::Literal, (L * sizeof(T)) / 16> make_literal(Us... values)
532 : : {
533 [ + - + - : 20 : ::wasm::Literal vectors[L] = { make_literal<T, 1, true>(values)... }; // fill multiple vectors with values
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
534 : :
535 [ - + - + : 20 : std::array<::wasm::Literal, (L * sizeof(T)) / 16> literals;
- + ]
536 [ + + + + : 84 : for (std::size_t idx = 0; idx < (L * sizeof(T)) / 16; ++idx) {
+ + ]
537 [ + - + - : 64 : auto vec = std::to_array(*reinterpret_cast<::wasm::Literal(*)[16 / sizeof(T)]>(vectors + idx * (16 / sizeof(T))));
+ - ]
538 [ + - + - : 64 : literals[idx] = ::wasm::Literal(vec);
+ - + - +
- + - ]
539 : 64 : }
540 : 20 : return literals;
541 [ + - + - : 20 : }
+ - ]
542 : :
543 : :
544 : : /*======================================================================================================================
545 : : * Exceptions
546 : : *====================================================================================================================*/
547 : :
548 : : #define M_EXCEPTION_LIST(X) \
549 : : X(invalid_escape_sequence) \
550 : : X(unreachable) \
551 : : X(failed_unittest_check)
552 : :
553 : : struct exception : backend_exception
554 : : {
555 : : #define DECLARE_ENUM(TYPE) TYPE,
556 : : enum exception_t : uint64_t {
557 : : M_EXCEPTION_LIST(DECLARE_ENUM)
558 : : };
559 : : #undef DECLARE_ENUM
560 : :
561 : : #define DECLARE_NAMES(TYPE) #TYPE,
562 : : static constexpr const char * const names_[] = {
563 : : M_EXCEPTION_LIST(DECLARE_NAMES)
564 : : };
565 : : #undef DECLARE_NAMES
566 : :
567 : : private:
568 : : exception_t type_;
569 : :
570 : : public:
571 [ # # ]: 0 : explicit exception(exception_t type, std::string message) : backend_exception(std::move(message)), type_(type) { }
572 : : };
573 : :
574 : :
575 : : /*======================================================================================================================
576 : : * Callback functions
577 : : *====================================================================================================================*/
578 : :
579 : : /** Reports a runtime error. The index to the filename, the line, and an optional message stored by the host is given
580 : : * by `args`. */
581 : : ::wasm::Literals insist_interpreter(::wasm::Literals &args);
582 : :
583 : : /** Throws an exception. The exception type id and the index to the filename, the line, and an optional message stored
584 : : * by the host is given by `args`. */
585 : : ::wasm::Literals throw_interpreter(::wasm::Literals &args);
586 : :
587 : : const std::map<::wasm::Name, std::function<::wasm::Literals(::wasm::Literals&)>> callback_functions = {
588 : : #define CALLBACK(NAME, FUNC) { NAME, FUNC },
589 : : CALLBACK("insist", insist_interpreter)
590 : : CALLBACK("throw", throw_interpreter)
591 : : #undef CALLBACK
592 : : };
593 : :
594 : :
595 : : /*======================================================================================================================
596 : : * GarbageCollectedData
597 : : *====================================================================================================================*/
598 : :
599 : : /** Helper struct for garbage collection done by the `Module`. Inherit from this struct, provide a c`tor expecting
600 : : * a `GarbageCollectedData&&` instance, and register the created struct in the module to garbage collect it
601 : : * automatically when the module is destroyed. */
602 : : struct GarbageCollectedData
603 : : {
604 : : friend struct Module;
605 : :
606 : : private:
607 : 2138 : GarbageCollectedData() = default;
608 : :
609 : : public:
610 : 2138 : GarbageCollectedData(GarbageCollectedData&&) = default;
611 : :
612 : 4276 : virtual ~GarbageCollectedData() { }
613 : : };
614 : :
615 : :
616 : : /*======================================================================================================================
617 : : * ConstantFolding
618 : : *====================================================================================================================*/
619 : :
620 : : /** Helper struct to perform constant folding at compile time. */
621 : : struct ConstantFolding
622 : : {
623 : : enum boolean_result_t {
624 : : UNDEF,
625 : : TRUE,
626 : : FALSE
627 : : };
628 : :
629 : : /** Tries to evaluate the given boolean expression \p expr using constant folding. Returns `UNDEF` if the
630 : : * expression cannot be evaluated at compile time, `TRUE` if the expression evaluates to `true` at compile time,
631 : : * and `FALSE` otherwise.
632 : : * Currently supported are only expressions consisting of a single boolean constant, a negation (tested using
633 : : * `eqZ` instruction) of a constant boolean expression, and conjunctions or disjunctions of constant boolean
634 : : * expressions. Expressions like `x == x` are currently not supported. */
635 : : static boolean_result_t EvalBoolean(const ::wasm::Expression *expr);
636 : : };
637 : :
638 : :
639 : : /*======================================================================================================================
640 : : * Module
641 : : *====================================================================================================================*/
642 : :
643 : : struct Module final
644 : : {
645 : : /*----- Friends --------------------------------------------------------------------------------------------------*/
646 : : friend struct Block;
647 : : friend struct BlockUser;
648 : : template<typename> friend struct Function;
649 : : template<typename, std::size_t> friend struct PrimitiveExpr;
650 : : template<typename, VariableKind, bool, std::size_t> friend class detail::variable_storage;
651 : : template<std::size_t> friend struct LocalBit;
652 : : friend struct Allocator;
653 : : template<typename> friend struct invoke_v8;
654 : :
655 : : private:
656 : : ///> counter to make block names unique
657 : : static inline std::atomic_uint NEXT_MODULE_ID_ = 0;
658 : :
659 : : ///> the unique ID for this `Module`
660 : : unsigned id_;
661 : : ///> counter to make block names unique
662 : : unsigned next_block_id_ = 0;
663 : : ///> counter to make function names unique
664 : : unsigned next_function_id_ = 0;
665 : : ///> counter to make global variable names unique
666 : : unsigned next_global_id_ = 0;
667 : : ///> counter to make if names unique
668 : : unsigned next_if_id_ = 0;
669 : : ///> counter to make loop names unique
670 : : unsigned next_loop_id_ = 0;
671 : : ///> the Binaryen Wasm module
672 : : ::wasm::Module module_;
673 : : ///> the Binaryen expression builder for the `module_`
674 : : ::wasm::Builder builder_;
675 : : ///> the currently active Binaryen block
676 : : ::wasm::Block *active_block_ = nullptr;
677 : : ///> the currently active Binaryen function
678 : : ::wasm::Function *active_function_ = nullptr;
679 : : ///> the main memory of the module
680 : : ::wasm::Memory *memory_ = nullptr;
681 : : ///> the virtual address space and its backed memory; only set if no `WasmContext` was created
682 : : std::unique_ptr<std::pair<memory::AddressSpace, memory::Memory>> vm_;
683 : : ///> the allocator
684 : : std::unique_ptr<Allocator> allocator_;
685 : : ///> stack of Binaryen branch targets
686 : : std::vector<branch_target_t> branch_target_stack_;
687 : : ///> filename, line, and an optional message for each emitted insist or exception throw
688 : : std::vector<std::tuple<const char*, unsigned, const char*>> messages_;
689 : : ///> this module's interface, if any
690 : : std::unique_ptr<::wasm::ModuleRunner::ExternalInterface> interface_;
691 : : ///> the per-function stacks of local bitmaps; used for local scalar boolean variables and NULL bits
692 : : std::vector<std::vector<LocalBitmap*>> local_bitmaps_stack_;
693 : : ///> the per-function stacks of local bitvectors; used for local vectorial boolean variables and NULL bits
694 : : std::vector<std::vector<LocalBitvector*>> local_bitvectors_stack_;
695 : : ///> mapping from handles to garbage collected data
696 : : std::unordered_map<void*, std::unique_ptr<GarbageCollectedData>> garbage_collected_data_;
697 : :
698 : : /*----- Thread-local instance ------------------------------------------------------------------------------------*/
699 : : private:
700 : : static thread_local std::unique_ptr<Module> the_module_;
701 : :
702 : : Module();
703 : : Module(const Module&) = delete;
704 : :
705 : : public:
706 : 1658 : static void Init() {
707 : 1658 : M_insist(not the_module_, "must not have a module yet");
708 [ + - ]: 1658 : the_module_ = std::unique_ptr<Module>(new Module());
709 : 1658 : }
710 : 1658 : static void Dispose() {
711 : 1658 : M_insist(bool(the_module_), "must have a module");
712 : 1658 : the_module_ = nullptr;
713 : 1658 : }
714 : 177163 : static Module & Get() {
715 : 177163 : M_insist(bool(the_module_), "must have a module");
716 : 177163 : return *the_module_;
717 : : }
718 : :
719 : : /*----- Access methods -------------------------------------------------------------------------------------------*/
720 : : /** Returns the ID of the current module. */
721 : 5758 : static unsigned ID() { return Get().id_; }
722 : :
723 : : /** Returns a unique block name in the current module. */
724 [ + - + - ]: 9144 : static std::string Unique_Block_Name(std::string prefix = "block") { return unique(prefix, Get().next_block_id_); }
725 : : /** Returns a unique function name in the current module. */
726 : 2009 : static std::string Unique_Function_Name(std::string prefix = "function") {
727 [ + - + - ]: 2009 : return unique(prefix, Get().next_function_id_);
728 : 0 : }
729 : : /** Returns a unique global name in the current module. */
730 : 5046 : static std::string Unique_Global_Name(std::string prefix = "global") {
731 [ + - + - ]: 5046 : return unique(prefix, Get().next_global_id_);
732 : 0 : }
733 : : /** Returns a unique if name in the current module. */
734 [ + - + - ]: 3040 : static std::string Unique_If_Name(std::string prefix = "if") { return unique(prefix, Get().next_if_id_); }
735 : : /** Returns a unique loop name in the current module. */
736 [ + - + - ]: 188 : static std::string Unique_Loop_Name(std::string prefix = "loop") { return unique(prefix, Get().next_loop_id_); }
737 : :
738 : : /** Returns the expression builder of the current module. */
739 : 66619 : static ::wasm::Builder & Builder() { return Get().builder_; }
740 : :
741 : : /** Returns the currently active block. */
742 : 9720 : static ::wasm::Block & Block() { return *M_notnull(Get().active_block_); }
743 : :
744 : : /** Returns the currently active function. */
745 : 3144 : static ::wasm::Function & Function() { return *M_notnull(Get().active_function_); }
746 : :
747 : : /** Returns the virtual address space. */
748 : : static memory::AddressSpace & Memory();
749 : :
750 : : /** Returns the allocator. */
751 : : static Allocator & Allocator();
752 : :
753 : : /** Validates that the module is well-formed. */
754 : : static bool Validate(bool verbose = true, bool global = true);
755 : :
756 : : /** Optimizes the module with the optimization level set to `level`. */
757 : : static void Optimize(int optimization_level);
758 : :
759 : : /** Sets the new active `::wasm::Block` and returns the previously active `::wasm::Block`. */
760 : 13504 : ::wasm::Block * set_active_block(::wasm::Block *block) { return std::exchange(active_block_, block); }
761 : : /** Sets the new active `::wasm::Function` and returns the previously active `::wasm::Function`. */
762 : 5672 : ::wasm::Function * set_active_function(::wasm::Function *fn) { return std::exchange(active_function_, fn); }
763 : :
764 : : /*----- Control flow ---------------------------------------------------------------------------------------------*/
765 : : /** An unsafe, i.e. statically-**un**typed, version of `Function::emit_return()`. */
766 : : void emit_return();
767 : : /** An unsafe, i.e. statically-**un**typed, version of `Function::emit_return(T&&)`. */
768 : : template<typename T, std::size_t L>
769 : : void emit_return(PrimitiveExpr<T, L> expr);
770 : : /** An unsafe, i.e. statically-**un**typed, version of `Function::emit_return(T&&)`. */
771 : : template<typename T, std::size_t L>
772 : : void emit_return(Expr<T, L> expr);
773 : :
774 : : void emit_break(std::size_t level = 1);
775 : : void emit_break(PrimitiveExpr<bool, 1> cond, std::size_t level = 1);
776 : :
777 : : void emit_continue(std::size_t level = 1);
778 : : void emit_continue(PrimitiveExpr<bool, 1> cond, std::size_t level = 1);
779 : :
780 : : template<typename T, std::size_t L>
781 : : PrimitiveExpr<T, L> emit_select(PrimitiveExpr<bool, 1> cond, PrimitiveExpr<T, L> tru, PrimitiveExpr<T, L> fals);
782 : : template<typename T, std::size_t L>
783 : : Expr<T, L> emit_select(PrimitiveExpr<bool, 1> cond, Expr<T, L> tru, Expr<T, L> fals);
784 : : template<typename T, std::size_t L>
785 : : requires (L > 1) and requires (PrimitiveExpr<int8_t, L> e) { e.template to<int_t<sizeof(T)>, L>(); }
786 : : PrimitiveExpr<T, L> emit_select(PrimitiveExpr<bool, L> cond, PrimitiveExpr<T, L> tru, PrimitiveExpr<T, L> fals);
787 : : template<typename T, std::size_t L>
788 : : requires (L > 1) and requires (PrimitiveExpr<int8_t, L> e) { e.template to<int_t<sizeof(T)>, L>(); }
789 : : Expr<T, L> emit_select(PrimitiveExpr<bool, L> cond, Expr<T, L> tru, Expr<T, L> fals);
790 : :
791 : : /*----- Shuffle --------------------------------------------------------------------------------------------------*/
792 : : /** Selects lanes of \p first and \p second in byte granularity depending on the indices specified by \p indices.
793 : : * Indices `i` in the range [0, L * sizeof(T)) select the `i`-th lane of `first`, indices `i` in the
794 : : * range [L * sizeof(T), 2 * L * sizeof(T)) select the `i-(L * sizeof(T))`-th lane of `second`, indices outside of
795 : : * these ranges result in undefined values. */
796 : : template<typename T, std::size_t L, std::size_t M>
797 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (M <= 16) and (M % sizeof(T) == 0)
798 : : PrimitiveExpr<T, M / sizeof(T)> emit_shuffle_bytes(PrimitiveExpr<T, L> first, PrimitiveExpr<T, L> second,
799 : : const std::array<uint8_t, M> &indices);
800 : : /** Selects lanes of \p first and \p second in lane granularity depending on the indices specified by \p indices.
801 : : * Indices `i` in the range [0, L) select the `i`-th lane of `first`, indices `i` in the range [L, 2 * L) select
802 : : * the `i-L`-th lane of `second`, indices outside of these ranges result in undefined values. */
803 : : template<typename T, std::size_t L, std::size_t M>
804 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (is_pow_2(M)) and (M * sizeof(T) <= 16)
805 : : PrimitiveExpr<T, M> emit_shuffle_lanes(PrimitiveExpr<T, L> first, PrimitiveExpr<T, L> second,
806 : : const std::array<uint8_t, M> &indices);
807 : : /** Selects lanes of \p first and \p second in byte granularity depending on the indices specified by \p indices.
808 : : * Indices `i` in the range [0, L * sizeof(T)) select the `i`-th lane of `first`, indices `i` in the
809 : : * range [L * sizeof(T), 2 * L * sizeof(T)) select the `i-(L * sizeof(T))`-th lane of `second`, indices outside of
810 : : * these ranges result in undefined values. */
811 : : template<typename T, std::size_t L, std::size_t M>
812 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (M <= 16) and (M % sizeof(T) == 0) and (sizeof(T) == 1)
813 : : Expr<T, M / sizeof(T)> emit_shuffle_bytes(Expr<T, L> first, Expr<T, L> second, const std::array<uint8_t, M> &indices);
814 : : /** Selects lanes of \p first and \p second in lane granularity depending on the indices specified by \p indices.
815 : : * Indices `i` in the range [0, L) select the `i`-th lane of `first`, indices `i` in the range [L, 2 * L) select
816 : : * the `i-L`-th lane of `second`, indices outside of these ranges result in undefined values. */
817 : : template<typename T, std::size_t L, std::size_t M>
818 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (is_pow_2(M)) and (M * sizeof(T) <= 16)
819 : : Expr<T, M> emit_shuffle_lanes(Expr<T, L> first, Expr<T, L> second, const std::array<uint8_t, M> &indices);
820 : :
821 : : /*----- Globals. -------------------------------------------------------------------------------------------------*/
822 : : template<dsl_primitive T, std::size_t L = 1, dsl_primitive... Us>
823 : : requires (L * sizeof(T) <= 16) and requires (Us... us) { make_literal<T, L>(us...); }
824 : 5883 : void emit_global(::wasm::Name name, bool is_mutable, Us... inits) {
825 : 5883 : ::wasm::Builder::Mutability mut = is_mutable ? ::wasm::Builder::Mutability::Mutable
826 : : : ::wasm::Builder::Mutability::Immutable;
827 [ + - # # : 5883 : ::wasm::Const *_init = builder_.makeConst(make_literal<T, L>(inits...));
+ - # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
828 : 5883 : auto global = builder_.makeGlobal(name, wasm_type<T, L>(), _init, mut);
829 [ - + # # : 5883 : module_.addGlobal(std::move(global));
- + # # #
# - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
830 : 5883 : }
831 : : template<dsl_primitive T, std::size_t L = 1, dsl_primitive... Us>
832 : : requires (L * sizeof(T) <= 16) and requires (Us... us) { make_literal<T, L>(us...); }
833 : 5046 : void emit_global(const std::array<::wasm::Name, 1> &names, bool is_mutable, Us... inits) {
834 : 5046 : emit_global<T, L>(names[0], is_mutable, inits...);
835 : 5046 : }
836 : : template<dsl_primitive T, std::size_t L = 1, std::size_t N, dsl_primitive... Us>
837 : : requires requires (Us... us) { { make_literal<T, L>(us...) } -> std::same_as<std::array<::wasm::Literal, N>>; }
838 : 0 : void emit_global(const std::array<::wasm::Name, N> &names, bool is_mutable, Us... inits) {
839 : 0 : ::wasm::Builder::Mutability mut = is_mutable ? ::wasm::Builder::Mutability::Mutable
840 : : : ::wasm::Builder::Mutability::Immutable;
841 : 0 : auto literals = make_literal<T, L>(inits...);
842 [ # # # # : 0 : for (std::size_t idx = 0; idx < N; ++idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
843 [ # # # # : 0 : ::wasm::Const *_init = builder_.makeConst(literals[idx]);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
844 [ # # # # : 0 : auto global = builder_.makeGlobal(names[idx], wasm_type<T, L>(), _init, mut);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
845 [ # # # # : 0 : module_.addGlobal(std::move(global));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
846 : 0 : }
847 : 0 : }
848 : : template<dsl_pointer_to_primitive T, std::size_t L = 1>
849 : 10 : void emit_global(::wasm::Name name, bool is_mutable, uint32_t init) {
850 : 10 : ::wasm::Builder::Mutability mut = is_mutable ? ::wasm::Builder::Mutability::Mutable
851 : : : ::wasm::Builder::Mutability::Immutable;
852 [ + - ]: 10 : ::wasm::Const *_init = builder_.makeConst(::wasm::Literal(init));
853 : 10 : auto global = builder_.makeGlobal(name, wasm_type<T, L>(), _init, mut);
854 [ - + ]: 10 : module_.addGlobal(std::move(global));
855 : 10 : }
856 : :
857 : : template<typename T, std::size_t L = 1>
858 : : requires (L * sizeof(T) <= 16)
859 : : PrimitiveExpr<T, L> get_global(const char *name);
860 : :
861 : : /*----- Imports & Exports ----------------------------------------------------------------------------------------*/
862 : : template<typename T, std::size_t L = 1>
863 : : requires (dsl_primitive<T> or dsl_pointer_to_primitive<T>) and
864 : : requires { M_CONSTEXPR_COND_UNCAPTURED(std::is_pointer_v<T>, (make_literal<T, L>(0)), (make_literal<T, L>(T()))); }
865 : 827 : void emit_import(const char *extern_name, const char *intern_name = nullptr)
866 : : {
867 [ + - # # ]: 1654 : ::wasm::Const *value = M_CONSTEXPR_COND(std::is_pointer_v<T>, builder_.makeConst(make_literal<T, L>(0)),
868 : : builder_.makeConst(make_literal<T, L>(T())));
869 [ # # - + ]: 827 : auto global = builder_.makeGlobal(intern_name ? intern_name : extern_name, wasm_type<T, L>(), M_notnull(value),
870 : : ::wasm::Builder::Mutability::Immutable);
871 [ # # + - : 827 : global->module = "imports";
+ - ]
872 [ # # + - : 827 : global->base = extern_name;
+ - ]
873 [ # # + - ]: 827 : module_.addGlobal(std::move(global));
874 : 827 : }
875 : :
876 : : /** Add function `name` with type `T` as import. */
877 : : template<typename T>
878 : : requires std::is_function_v<T> and requires { wasm_type<T, 1>(); }
879 : 4143 : void emit_function_import(const char *name) {
880 [ + - - + : 4143 : auto func = module_.addFunction(builder_.makeFunction(name, wasm_type<T, 1>(), {}));
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
881 : 4143 : func->module = "imports";
882 : 4143 : func->base = name;
883 : 4143 : }
884 : :
885 : : /** Add function `name` as export. */
886 : 827 : void emit_function_export(const char *name) {
887 [ + - ]: 827 : module_.addExport(builder_.makeExport(name, name, ::wasm::ExternalKind::Function));
888 : 827 : }
889 : :
890 : : /*----- Function calls -------------------------------------------------------------------------------------------*/
891 : : template<typename ReturnType, typename... ParamTypes, std::size_t... ParamLs>
892 : : requires std::is_void_v<ReturnType>
893 : : void emit_call(const char *fn, PrimitiveExpr<ParamTypes, ParamLs>... args);
894 : :
895 : : template<typename ReturnType, std::size_t ReturnL = 1, typename... ParamTypes, std::size_t... ParamLs>
896 : : requires dsl_primitive<ReturnType> or dsl_pointer_to_primitive<ReturnType>
897 : : PrimitiveExpr<ReturnType, ReturnL> emit_call(const char *fn, PrimitiveExpr<ParamTypes, ParamLs>... args);
898 : :
899 : : /*----- Runtime checks and throwing exceptions -------------------------------------------------------------------*/
900 : : template<std::size_t L>
901 : : void emit_insist(PrimitiveExpr<bool, L> cond, const char *filename, unsigned line, const char *msg);
902 : : template<>
903 : : void emit_insist<1>(PrimitiveExpr<bool, 1> cond, const char *filename, unsigned line, const char *msg);
904 : :
905 : : void emit_throw(exception::exception_t type, const char *filename, unsigned line, const char *msg);
906 : :
907 : 0 : const std::tuple<const char*, unsigned, const char*> & get_message(std::size_t idx) const {
908 : 0 : return messages_.at(idx);
909 : : }
910 : :
911 : : /*----- Garbage collected data -----------------------------------------------------------------------------------*/
912 : : /** Adds and returns an instance of \tparam C, which will be created by calling its c`tor with an
913 : : * `GarbageCollectedData&&` instance and the forwarded \p args, to `this` `Module`s garbage collection using the
914 : : * unique caller handle \p handle. */
915 : : template<class C, typename... Args>
916 : 2138 : C & add_garbage_collected_data(void *handle, Args... args) {
917 [ - + - + : 2138 : auto it = garbage_collected_data_.template try_emplace(
# # # # ]
918 : : /* key= */ handle,
919 [ + - + - : 2138 : /* value= */ std::make_unique<C>(GarbageCollectedData(), std::forward<Args>(args)...)
# # # # ]
920 : 2138 : ).first;
921 : 2138 : return as<C>(*it->second);
922 : 0 : }
923 : :
924 : : /*----- Interpretation & Debugging -------------------------------------------------------------------------------*/
925 : : ::wasm::ModuleRunner::ExternalInterface * get_mock_interface(::wasm::GlobalValueSet imports = {});
926 : :
927 : : /** Create an instance of this module. Can be used for interpretation and debugging. */
928 : 829 : ::wasm::ModuleRunner instantiate(::wasm::GlobalValueSet imports = {}) {
929 [ + - - + ]: 829 : return ::wasm::ModuleRunner(module_, get_mock_interface(std::move(imports)));
930 : 0 : }
931 : :
932 : : /*----- Module settings ------------------------------------------------------------------------------------------*/
933 : : void set_feature(::wasm::FeatureSet feature, bool value) { module_.features.set(feature, value); }
934 : :
935 : : /** Returns the binary representation of `module_` in a freshly allocated memory. The caller must dispose of this
936 : : * memory. */
937 : : std::pair<uint8_t*, std::size_t> binary();
938 : :
939 : : private:
940 : : void create_local_bitmap_stack();
941 : : void create_local_bitvector_stack();
942 : : void dispose_local_bitmap_stack();
943 : : void dispose_local_bitvector_stack();
944 : : public:
945 : : template<std::size_t L = 1>
946 : : requires (L > 0) and (L <= 16)
947 : : LocalBit<L> allocate_bit();
948 : :
949 : 188 : void push_branch_targets(::wasm::Name brk, ::wasm::Name continu) {
950 : 188 : branch_target_stack_.emplace_back(brk, continu, nullptr);
951 : 188 : }
952 : :
953 : : void push_branch_targets(::wasm::Name brk, ::wasm::Name continu, PrimitiveExpr<bool, 1> condition);
954 : :
955 : 288 : branch_target_t pop_branch_targets() {
956 : 288 : auto top = branch_target_stack_.back();
957 : 288 : branch_target_stack_.pop_back();
958 : 288 : return top;
959 : : }
960 : :
961 : : const branch_target_t & current_branch_targets() const { return branch_target_stack_.back(); }
962 : :
963 : : /*----- Printing -------------------------------------------------------------------------------------------------*/
964 : : public:
965 : 0 : friend std::ostream & operator<<(std::ostream &out, const Module &M) {
966 : 0 : out << "Module\n";
967 : :
968 : 0 : out << " currently active block: ";
969 [ # # ]: 0 : if (M.active_block_) {
970 [ # # ]: 0 : if (M.active_block_->name.is())
971 : 0 : out << '"' << M.active_block_->name << '"';
972 : : else
973 : 0 : out << "<anonymous block>";
974 : 0 : } else {
975 : 0 : out << "none";
976 : : }
977 : 0 : out << '\n';
978 : :
979 : : // out << " currently active function: ";
980 : : // if (M.active_function_) {
981 : : // out << '"' << M.active_function_->name << '"';
982 : : // } else {
983 : : // out << "none";
984 : : // }
985 : : // out << '\n';
986 : :
987 : 0 : return out;
988 : : }
989 : :
990 : 0 : void dump(std::ostream &out) const { out << *this << std::endl; }
991 : : void dump() const { dump(std::cerr); }
992 : :
993 : 0 : void dump_all(std::ostream &out) { out << module_ << std::endl; }
994 : 0 : void dump_all() { dump_all(std::cerr); }
995 : : };
996 : :
997 : :
998 : : /*======================================================================================================================
999 : : * Block
1000 : : *====================================================================================================================*/
1001 : :
1002 : : /** Represents a code block, i.e. a sequential sequence of code. Necessary to compose conditional control flow and
1003 : : * useful for simultaneous code generation at several locations. */
1004 : : struct Block final
1005 : : {
1006 : : template<typename T> friend struct Function; // to get ::wasm::Block of body
1007 : : friend struct BlockUser; // to access `Block` internals
1008 : : friend struct If; // to get ::wasm::Block for *then* and *else* part
1009 : : friend struct Loop; // to get ::wasm::Block for loop body
1010 : : friend struct DoWhile; // to get ::wasm::Block for loop body
1011 : :
1012 : : private:
1013 : : ///> this block, can be `nullptr` if default-constructed or the block has already been attached
1014 : 30 : ::wasm::Block *this_block_ = nullptr;
1015 : : ///> the parent block, before this block was created
1016 : 9188 : ::wasm::Block *parent_block_ = nullptr;
1017 : : ///> whether this block attaches itself to its parent block
1018 : 30 : bool attach_to_parent_ = false;
1019 : :
1020 : : public:
1021 : 30 : friend void swap(Block &first, Block &second) {
1022 : : using std::swap;
1023 : 30 : swap(first.this_block_, second.this_block_);
1024 : 30 : swap(first.parent_block_, second.parent_block_);
1025 : 30 : swap(first.attach_to_parent_, second.attach_to_parent_);
1026 : 30 : }
1027 : :
1028 : : private:
1029 : 60 : Block() = default;
1030 : :
1031 : : /** Create a new `Block` for a given `::wasm::Block`. */
1032 : 9158 : Block(::wasm::Block *block, bool attach_to_parent)
1033 : 9158 : : this_block_(M_notnull(block))
1034 : 9158 : , attach_to_parent_(attach_to_parent)
1035 : : {
1036 [ + + ]: 9158 : if (attach_to_parent_) {
1037 : 2 : parent_block_ = Module::Get().active_block_;
1038 [ - + ]: 2 : M_insist(not attach_to_parent_ or parent_block_, "can only attach to parent if there is a parent block");
1039 : 2 : }
1040 : 9158 : }
1041 : :
1042 : : public:
1043 : : /** Create an anonymous `Block`. */
1044 : 14 : explicit Block(bool attach_to_parent) : Block(Module::Builder().makeBlock(), attach_to_parent) { }
1045 : : /** Create a named `Block` and set it *active* in the current `Module`. */
1046 : 9144 : explicit Block(std::string name, bool attach_to_parent)
1047 [ + - + - : 9144 : : Block(Module::Builder().makeBlock(Module::Unique_Block_Name(name)), attach_to_parent)
+ - + - ]
1048 : 9144 : { }
1049 : : /** Create a named `Block` and set it *active* in the current `Module`. */
1050 [ + - - + ]: 40 : explicit Block(const char *name, bool attach_to_parent) : Block(std::string(name), attach_to_parent) { }
1051 : :
1052 [ + - ]: 30 : Block(Block &&other) : Block() { swap(*this, other); }
1053 : :
1054 : 9188 : ~Block() {
1055 [ + + + + ]: 9188 : if (this_block_ and attach_to_parent_)
1056 [ + - + - ]: 2 : attach_to(*M_notnull(parent_block_));
1057 : 9188 : }
1058 : :
1059 : : Block & operator=(Block &&other) { swap(*this, other); return *this; }
1060 : :
1061 : : private:
1062 : 11332 : ::wasm::Block & get() const { return *M_notnull(this_block_); }
1063 : : ::wasm::Block & previous() const { return *M_notnull(parent_block_); }
1064 : :
1065 : 44 : void attach_to(::wasm::Block &other) {
1066 : 44 : other.list.push_back(this_block_);
1067 : 44 : this_block_ = nullptr;
1068 : 44 : }
1069 : :
1070 : : public:
1071 : : bool has_name() const { return bool(get().name); }
1072 : : std::string name() const { M_insist(has_name()); return get().name.toString(); }
1073 : :
1074 : : /** Returns whether this `Block` is empty, i.e. contains to expressions. */
1075 : 22 : bool empty() const { return get().list.empty(); }
1076 : :
1077 : : /** Attaches this `Block` to the given `Block` \p other. */
1078 : : void attach_to(Block &other) {
1079 : : M_insist(not attach_to_parent_, "cannot explicitly attach if attach_to_parent is true");
1080 : : attach_to(*M_notnull(other.this_block_));
1081 : : }
1082 : :
1083 : : /** Attaches this `Block` to the `wasm::Block` currently active in the `Module`. */
1084 : 42 : void attach_to_current() {
1085 : 42 : M_insist(not attach_to_parent_, "cannot explicitly attach if attach_to_parent is true");
1086 : 42 : attach_to(Module::Block());
1087 : 42 : }
1088 : :
1089 : : /** Emits a jump to the end of this `Block`. */
1090 : 0 : void go_to() const { Module::Block().list.push_back(Module::Builder().makeBreak(get().name)); }
1091 : : /** Emits a jump to the end of this `Block` iff `cond` is fulfilled. */
1092 : : void go_to(PrimitiveExpr<bool, 1> cond) const;
1093 : :
1094 : : friend std::ostream & operator<<(std::ostream &out, const Block &B) {
1095 : : out << "vvvvvvvvvv block";
1096 : : if (B.has_name())
1097 : : out << " \"" << B.name() << '"';
1098 : : out << " starts here vvvvvvvvvv\n";
1099 : :
1100 : : for (auto expr : B.get().list)
1101 : : out << *expr << '\n';
1102 : :
1103 : : out << "^^^^^^^^^^^ block";
1104 : : if (B.has_name())
1105 : : out << " \"" << B.name() << '"';
1106 : : out << " ends here ^^^^^^^^^^^\n";
1107 : :
1108 : : return out;
1109 : : }
1110 : :
1111 : : void dump(std::ostream &out) const { out << *this; out.flush(); }
1112 : : void dump() const { dump(std::cerr); }
1113 : : };
1114 : :
1115 : : /** A helper class to *use* a `Block`, thereby setting the `Block` active for code generation. When the `BlockUser` is
1116 : : * destructed, restores the previously active block for code generation. */
1117 : : struct BlockUser
1118 : : {
1119 : : private:
1120 : : Block &block_; ///< the block to use (for code gen)
1121 : 6752 : ::wasm::Block *old_block_ = nullptr; ///< the previously active, now old block
1122 : :
1123 : : public:
1124 : 6752 : BlockUser(Block &block) : block_(block) {
1125 : 6752 : old_block_ = Module::Get().set_active_block(block_.this_block_); // set active block
1126 : 6752 : }
1127 : :
1128 [ + - ]: 6752 : ~BlockUser() { Module::Get().set_active_block(old_block_); } // restore previously active block
1129 : : };
1130 : :
1131 : :
1132 : : /*======================================================================================================================
1133 : : * Function
1134 : : *====================================================================================================================*/
1135 : :
1136 : : /** Represents a Wasm function. It is templated with return type and parameter types. This enables us to access
1137 : : * parameters with their proper types. */
1138 : : template<typename>
1139 : : struct Function;
1140 : :
1141 : : template<typename ReturnType, typename... ParamTypes, std::size_t ReturnL, std::size_t... ParamLs>
1142 : : requires ((std::is_void_v<ReturnType> and (ReturnL == 1)) or
1143 : : requires { typename PrimitiveExpr<ReturnType, ReturnL>; }) and
1144 : : (not dsl_primitive<ReturnType> or (ReturnL * sizeof(ReturnType) <= 16)) and // must fit in single register
1145 : : (requires { typename PrimitiveExpr<ParamTypes, ParamLs>; } and ...) and
1146 : : ((not dsl_primitive<ParamTypes> or (ParamLs * sizeof(ParamTypes) <= 16)) and ...) // must fit in single register
1147 : : struct Function<PrimitiveExpr<ReturnType, ReturnL>(PrimitiveExpr<ParamTypes, ParamLs>...)>
1148 : : {
1149 : : template<typename> friend struct FunctionProxy;
1150 : :
1151 : : ///> the type of the function
1152 : : using type = ReturnType(ParamTypes...);
1153 : : ///> the DSL type of the function
1154 : : using dsl_type = PrimitiveExpr<ReturnType, ReturnL>(PrimitiveExpr<ParamTypes, ParamLs>...);
1155 : : ///> the return type of the function
1156 : : using return_type = ReturnType;
1157 : : ///> the amount of parameters of the function
1158 : : static constexpr std::size_t PARAMETER_COUNT = sizeof...(ParamTypes);
1159 : :
1160 : : /*------------------------------------------------------------------------------------------------------------------
1161 : : * Parameter helper types
1162 : : *----------------------------------------------------------------------------------------------------------------*/
1163 : : private:
1164 : : template<typename... Ts, std::size_t... Ls, std::size_t... Is>
1165 : : requires (sizeof...(Ts) == sizeof...(Ls)) and (sizeof...(Ts) == sizeof...(Is))
1166 : : std::tuple<Parameter<Ts, Ls>...> make_parameters_helper(std::index_sequence<Is...>) {
1167 : : return std::make_tuple<Parameter<Ts, Ls>...>(
1168 : : (Parameter<Ts, Ls>(Is), ...)
1169 : : );
1170 : : }
1171 : :
1172 : : /** Creates a `std::tuple` with statically typed fields, one per parameter. */
1173 : : template<typename... Ts, std::size_t... Ls, typename Indices = std::make_index_sequence<sizeof...(Ts)>>
1174 : : requires (sizeof...(Ts) == sizeof...(Ls))
1175 : : std::tuple<Parameter<Ts, Ls>...> make_parameters() { return make_parameters_helper<Ts..., Ls...>(Indices{}); }
1176 : :
1177 : : /** Provides an alias `type` with the type of the \tparam I -th parameter. */
1178 : : template<std::size_t I, typename... Ts>
1179 : : struct parameter_type;
1180 : : template<std::size_t I, typename T, typename... Ts>
1181 : : struct parameter_type<I, T, Ts...>
1182 : : {
1183 : : static_assert(I <= sizeof...(Ts), "parameter index out of range");
1184 : : using type = typename parameter_type<I - 1, Ts...>::type;
1185 : : };
1186 : : template<typename T, typename... Ts>
1187 : : struct parameter_type<0, T, Ts...>
1188 : : {
1189 : : using type = T;
1190 : : };
1191 : : /** Convenience alias for `parameter_type::type`. */
1192 : : template<std::size_t I>
1193 : : using parameter_type_t = typename parameter_type<I, ParamTypes...>::type;
1194 : :
1195 : : /** Provides a value `value` with the number of SIMD lanes of the \tparam I -th parameter. */
1196 : : template<std::size_t I, std::size_t... Ls>
1197 : : struct parameter_num_simd_lanes;
1198 : : template<std::size_t I, std::size_t L, std::size_t... Ls>
1199 : : struct parameter_num_simd_lanes<I, L, Ls...>
1200 : : {
1201 : : static_assert(I <= sizeof...(Ls), "parameter index out of range");
1202 : : static constexpr std::size_t value = parameter_num_simd_lanes<I - 1, Ls...>::value;
1203 : : };
1204 : : template<std::size_t L, std::size_t... Ls>
1205 : : struct parameter_num_simd_lanes<0, L, Ls...>
1206 : : {
1207 : : static constexpr std::size_t value = L;
1208 : : };
1209 : : /** Convenience alias for `parameter_num_simd_lanes::value`. */
1210 : : template<std::size_t I>
1211 : : static constexpr std::size_t parameter_num_simd_lanes_v = parameter_num_simd_lanes<I, ParamLs...>::value;
1212 : :
1213 : : private:
1214 : : ::wasm::Name name_; ///< the *unique* name of this function
1215 : : Block body_; ///< the function body
1216 : : ///> the `::wasm::Function` implementing this function
1217 : 2836 : ::wasm::Function *this_function_ = nullptr;
1218 : : ///> the previously active `::wasm::Function` (may be `nullptr`)
1219 : 2836 : ::wasm::Function *previous_function_ = nullptr;
1220 : :
1221 : : public:
1222 : : friend void swap(Function &first, Function &second) {
1223 : : using std::swap;
1224 : : swap(first.name_, second.name_);
1225 : : swap(first.body_, second.body_);
1226 : : swap(first.this_function_, second.this_function_);
1227 : : swap(first.previous_function_, second.previous_function_);
1228 : : }
1229 : :
1230 : : private:
1231 : : Function() = default;
1232 : :
1233 : : public:
1234 : : /** Constructs a fresh `Function` and expects a unique \p name. To be called by `FunctionProxy`. */
1235 : 2836 : Function(const std::string &name)
1236 : 2836 : : name_(name)
1237 [ + - + - : 2836 : , body_(name + ".body", /* attach_to_parent= */ false)
+ - + - +
- + - # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
+ - # # +
- # # + -
# # + - #
# # # # #
# # # # ]
1238 : : {
1239 : : /*----- Set block return type for non-`void` functions. -----*/
1240 : : if constexpr (not std::is_void_v<ReturnType>)
1241 [ + - + - : 2413 : body_.get().type = wasm_type<ReturnType, ReturnL>();
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# + - + -
# # # # #
# # # # #
# # # # #
# ]
1242 : :
1243 : : /*----- Create Binaryen function. -----*/
1244 [ + - - + : 5672 : auto fn = Module::Builder().makeFunction(
+ - - + +
- - + + -
- + + - -
+ + - - +
# # # # #
# # # + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ # # # #
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
# # # # +
- - + # #
# # + - -
+ # # # #
+ - - + #
# # # + -
- + # # #
# # # # #
# # # # #
# # # # #
# # ]
1245 [ + - + - : 2836 : /* name= */ name,
+ - + - +
- + - # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
+ - # # +
- # # + -
# # + - #
# # # # #
# # # # ]
1246 [ + - + - : 2836 : /* type= */ wasm_type<dsl_type, 1>(),
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
1247 : 2836 : /* vars= */ std::vector<::wasm::Type>{}
1248 : : );
1249 [ + - + - : 2836 : fn->body = &body_.get(); // set function body
+ - + - +
- + - # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
+ - # # +
- # # + -
# # + - #
# # # # #
# # # # ]
1250 [ + - + - : 2836 : this_function_ = Module::Get().module_.addFunction(std::move(fn));
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
1251 [ + - + - : 2836 : M_insist(this_function_->getNumParams() == PARAMETER_COUNT);
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
1252 [ + - + - : 2836 : Module::Get().create_local_bitmap_stack();
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
1253 [ + - + - : 2836 : Module::Get().create_local_bitvector_stack();
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
1254 : :
1255 : : /*----- Set this function active in the `Module`. -----*/
1256 [ + - + - : 2836 : previous_function_ = Module::Get().set_active_function(this_function_);
+ - + - +
- + - + -
# # # # +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # + - #
# + - # #
+ - # # +
- # # # #
# # # # #
# ]
1257 : 2836 : }
1258 : :
1259 : : Function(const Function&) = delete;
1260 : : Function(Function &&other) : Function() { swap(*this, other); }
1261 : :
1262 : 2836 : ~Function() {
1263 : : if constexpr (not std::is_void_v<ReturnType>)
1264 [ + - + - : 2413 : body_.get().list.push_back(Module::Builder().makeUnreachable());
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # #
# # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # # #
# # + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- # # # #
# # # # +
- + - + -
+ - # # #
# # # # #
+ - + - +
- + - # #
# # # # #
# + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1265 [ + - + - : 2836 : Module::Get().dispose_local_bitmap_stack();
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
1266 [ + - + - : 2836 : Module::Get().dispose_local_bitvector_stack();
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # +
- + - # #
# # + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
1267 : : /*----- Restore previously active function in the `Module`. -----*/
1268 [ + - + - : 2836 : Module::Get().set_active_function(previous_function_);
+ - + - +
- + - # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
+ - # # +
- # # + -
# # + - #
# # # # #
# # # # ]
1269 : 2836 : }
1270 : :
1271 : : Function & operator=(Function &&other) { swap(*this, other); return *this; }
1272 : :
1273 : : public:
1274 : : /** Returns the body of this function. */
1275 : 2836 : Block & body() { return body_; }
1276 : : /** Returns the body of this function. */
1277 : : const Block & body() const { return body_; }
1278 : :
1279 : : /** Returns the name of this function. */
1280 : : std::string name() const { return name_.toString(); }
1281 : :
1282 : : /** Returns all parameters of this function. */
1283 : : std::tuple<Parameter<ParamTypes, ParamLs>...> parameters() { return make_parameters<ParamTypes..., ParamLs...>(); }
1284 : :
1285 : : /** Returns the \tparam I -th parameter, statically typed via `parameter_type_t`. */
1286 : : template<std::size_t I>
1287 : 647 : Parameter<parameter_type_t<I>, parameter_num_simd_lanes_v<I>> parameter() {
1288 : 647 : return Parameter<parameter_type_t<I>, parameter_num_simd_lanes_v<I>>(I);
1289 : : }
1290 : :
1291 : : /** Emits a return instruction returning `void`. */
1292 : 2 : void emit_return() requires std::is_void_v<ReturnType> {
1293 : 2 : Module::Block().list.push_back(Module::Builder().makeReturn());
1294 : 2 : }
1295 : :
1296 : : /** Emits a return instruction returning `PrimitiveExpr` constructed from \p t of type \tparam T. */
1297 : : template<primitive_convertible T>
1298 : : requires (not std::is_void_v<ReturnType>) and
1299 : : requires (T &&t) { PrimitiveExpr<ReturnType, ReturnL>(primitive_expr_t<T>(std::forward<T>(t))); }
1300 : 1773 : void emit_return(T &&t) {
1301 [ + - + - : 1773 : PrimitiveExpr<ReturnType, ReturnL> value(primitive_expr_t<T>(std::forward<T>(t)));
+ - + - ]
1302 [ + - + - : 1773 : Module::Get().emit_return(value);
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1303 : 1773 : }
1304 : :
1305 : : /** Emits a return instruction returning `PrimitiveExpr` constructed from \p t of type \tparam T. Checks that
1306 : : * \p t is `NOT NULL`. */
1307 : : template<expr_convertible T>
1308 : : requires (not std::is_void_v<ReturnType>) and (not primitive_convertible<T>) and
1309 : : requires (T t) { Expr<ReturnType, ReturnL>(expr_t<T>(std::forward<T>(t))); }
1310 : 684 : void emit_return(T &&t)
1311 : : {
1312 [ + - + - : 684 : Expr<ReturnType, ReturnL> expr(expr_t<T>(std::forward<T>(t)));
+ - + - +
- + - + -
+ - + - +
- ]
1313 [ + - + - : 684 : Module::Get().emit_return(expr);
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - + - -
+ ]
1314 : 684 : }
1315 : :
1316 : : private:
1317 : : ::wasm::Function & get() const { return *M_notnull(this_function_); }
1318 : :
1319 : : public:
1320 : : friend std::ostream & operator<<(std::ostream &out, const Function &Fn) {
1321 : : out << "function \"" << Fn.name() << "\" : ";
1322 : : if constexpr (PARAMETER_COUNT)
1323 : : out << print_types<param_pack_t<ParamTypes...>, ParamLs...>{};
1324 : : else
1325 : : out << typeid(void).name();
1326 : : out << " -> " << print_types<param_pack_t<ReturnType>, ReturnL>{} << '\n';
1327 : :
1328 : : if (not Fn.get().vars.empty()) {
1329 : : out << " " << Fn.get().getNumVars() << " local variables:";
1330 : : for (::wasm::Index i = 0, end = Fn.get().getNumVars(); i != end; ++i)
1331 : : out << " [" << i << "] " << Fn.get().vars[i];
1332 : : out << '\n';
1333 : : }
1334 : :
1335 : : out << Fn.body();
1336 : : return out;
1337 : : }
1338 : :
1339 : : void dump(std::ostream &out) const { out << *this; out.flush(); }
1340 : : void dump() const { dump(std::cerr); }
1341 : : };
1342 : :
1343 : : template<typename ReturnType, typename... ParamTypes>
1344 : : requires (std::is_void_v<ReturnType> or dsl_primitive<ReturnType> or dsl_pointer_to_primitive<ReturnType>) and
1345 : : ((dsl_primitive<ParamTypes> or dsl_pointer_to_primitive<ParamTypes>) and ...)
1346 : : struct Function<ReturnType(ParamTypes...)> : Function<PrimitiveExpr<ReturnType, 1>(PrimitiveExpr<ParamTypes, 1>...)>
1347 : : {
1348 : : using Function<PrimitiveExpr<ReturnType, 1>(PrimitiveExpr<ParamTypes, 1>...)>::Function;
1349 : : };
1350 : :
1351 : : template<typename... ParamTypes, std::size_t... ParamLs>
1352 : : struct Function<void(PrimitiveExpr<ParamTypes, ParamLs>...)>
1353 : : : Function<PrimitiveExpr<void, 1>(PrimitiveExpr<ParamTypes, ParamLs>...)>
1354 : : {
1355 : : using Function<PrimitiveExpr<void, 1>(PrimitiveExpr<ParamTypes, ParamLs>...)>::Function;
1356 : : };
1357 : :
1358 : :
1359 : : /*======================================================================================================================
1360 : : * FunctionProxy
1361 : : *====================================================================================================================*/
1362 : :
1363 : : /** A handle to create a `Function` and to create invocations of that function. Provides `operator()()` to emit a
1364 : : * function call by issuing a C-style call. The class is template typed with the function signature, allowing us to
1365 : : * perform static type checking of arguments and the returned value at call sites. */
1366 : : template<typename>
1367 : : struct FunctionProxy;
1368 : :
1369 : : template<typename ReturnType, typename... ParamTypes, std::size_t ReturnL, std::size_t... ParamLs>
1370 : : requires ((std::is_void_v<ReturnType> and (ReturnL == 1)) or
1371 : : requires { typename PrimitiveExpr<ReturnType, ReturnL>; }) and
1372 : : (not dsl_primitive<ReturnType> or (ReturnL * sizeof(ReturnType) <= 16)) and // must fit in single register
1373 : : (requires { typename PrimitiveExpr<ParamTypes, ParamLs>; } and ...) and
1374 : : ((not dsl_primitive<ParamTypes> or (ParamLs * sizeof(ParamTypes) <= 16)) and ...) // must fit in single register
1375 : : struct FunctionProxy<PrimitiveExpr<ReturnType, ReturnL>(PrimitiveExpr<ParamTypes, ParamLs>...)>
1376 : : {
1377 : : using type = ReturnType(ParamTypes...);
1378 : : using dsl_type = PrimitiveExpr<ReturnType, ReturnL>(PrimitiveExpr<ParamTypes, ParamLs>...);
1379 : :
1380 : : private:
1381 : : std::string name_; ///< the unique name of the `Function`
1382 : :
1383 : : public:
1384 : : FunctionProxy() = delete;
1385 [ + - + - : 2009 : FunctionProxy(std::string name) : name_(Module::Unique_Function_Name(name)) { }
+ - + - +
- + - # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- ]
1386 [ + - - + : 2009 : FunctionProxy(const char *name) : FunctionProxy(std::string(name)) { }
+ - - + +
- - + + -
- + + - -
+ # # # #
# # # # +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ ]
1387 : :
1388 : 325 : FunctionProxy(FunctionProxy&&) = default;
1389 : :
1390 : 0 : FunctionProxy & operator=(FunctionProxy&&) = default;
1391 : :
1392 : : const std::string & name() const { return name_; }
1393 : 827 : const char * c_name() const { return name_.c_str(); }
1394 : :
1395 : 2009 : Function<dsl_type> make_function() const { return Function<dsl_type>(name_); }
1396 : :
1397 : : /*----- Overload operator() to emit function calls ---------------------------------------------------------------*/
1398 : : /** Call function returning `void` with parameters \p args of types \tparam Args. */
1399 : : template<typename... Args>
1400 : : requires std::is_void_v<ReturnType> and
1401 : : requires (Args&&... args) { (PrimitiveExpr<ParamTypes, ParamLs>(std::forward<Args>(args)), ...); }
1402 : 2078 : void operator()(Args&&... args) const {
1403 [ + - # # : 2078 : operator()(PrimitiveExpr<ParamTypes, ParamLs>(std::forward<Args>(args))...);
# # # # ]
1404 : 2078 : }
1405 : :
1406 : : /** Call function returning `void` with parameters \p args of `PrimitiveExpr` type. */
1407 : 2132 : void operator()(PrimitiveExpr<ParamTypes, ParamLs>... args) const requires std::is_void_v<ReturnType> {
1408 [ + - + - : 4264 : Module::Block().list.push_back(
+ - + - +
- # # # #
+ - + - +
- + - + -
+ - # # ]
1409 [ + - + - : 2132 : Module::Builder().makeCall(name_, { args.expr()... }, wasm_type<ReturnType, ReturnL>())
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # #
# # # ]
1410 : : );
1411 : 2132 : }
1412 : :
1413 : : /** Call function returning non-`void` with parameters \p args of types \tparam Args. */
1414 : : template<typename... Args>
1415 : : requires (not std::is_void_v<ReturnType>) and
1416 : : requires (Args&&... args) { (PrimitiveExpr<ParamTypes, ParamLs>(std::forward<Args>(args)), ...); }
1417 : 68 : PrimitiveExpr<ReturnType, ReturnL> operator()(Args&&... args) const {
1418 [ + - + - : 68 : return operator()(PrimitiveExpr<ParamTypes, ParamLs>(std::forward<Args>(args))...);
+ - + - -
+ + - + -
+ - - + +
- + - +
- ]
1419 : 0 : }
1420 : :
1421 : : /** Call function returning non-`void` with parameters \p args of `PrimitiveExpr` type. */
1422 : : PrimitiveExpr<ReturnType, ReturnL>
1423 : 865 : operator()(PrimitiveExpr<ParamTypes, ParamLs>... args) const requires (not std::is_void_v<ReturnType>) {
1424 [ - + - + : 865 : return PrimitiveExpr<ReturnType, ReturnL>(
- + - + -
+ - + - +
- + - + -
+ - + - +
- + # # -
+ - + - +
# # - + -
+ # # # #
- + - + #
# # # - +
# # - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ # # ]
1425 [ + - + - : 865 : Module::Builder().makeCall(name_, { args.expr()... }, wasm_type<ReturnType, ReturnL>())
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # + - +
- + - + -
+ - + - #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - # #
# # # # #
# # # # #
+ - + - #
# # # # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # #
# ]
1426 : : );
1427 : 0 : }
1428 : : };
1429 : :
1430 : : template<typename ReturnType, typename... ParamTypes>
1431 : : requires (std::is_void_v<ReturnType> or dsl_primitive<ReturnType> or dsl_pointer_to_primitive<ReturnType>) and
1432 : : ((dsl_primitive<ParamTypes> or dsl_pointer_to_primitive<ParamTypes>) and ...)
1433 : : struct FunctionProxy<ReturnType(ParamTypes...)>
1434 : : : FunctionProxy<PrimitiveExpr<ReturnType, 1>(PrimitiveExpr<ParamTypes, 1>...)>
1435 : : {
1436 : : using FunctionProxy<PrimitiveExpr<ReturnType, 1>(PrimitiveExpr<ParamTypes, 1>...)>::FunctionProxy;
1437 : : };
1438 : :
1439 : : template<typename... ParamTypes, std::size_t... ParamLs>
1440 : : struct FunctionProxy<void(PrimitiveExpr<ParamTypes, ParamLs>...)>
1441 : : : FunctionProxy<PrimitiveExpr<void, 1>(PrimitiveExpr<ParamTypes, ParamLs>...)>
1442 : : {
1443 : : using FunctionProxy<PrimitiveExpr<void, 1>(PrimitiveExpr<ParamTypes, ParamLs>...)>::FunctionProxy;
1444 : : };
1445 : :
1446 : :
1447 : : /*======================================================================================================================
1448 : : * PrimitiveExpr
1449 : : *====================================================================================================================*/
1450 : :
1451 : : template<typename T, typename U, std::size_t L>
1452 : : concept arithmetically_combinable = dsl_primitive<T> and dsl_primitive<U> and have_common_type<T, U> and
1453 : : requires (PrimitiveExpr<T, L> e) { PrimitiveExpr<common_type_t<T, U>, L>(e); } and
1454 : : requires (PrimitiveExpr<U, L> e) { PrimitiveExpr<common_type_t<T, U>, L>(e); };
1455 : :
1456 : : /** Specialization of `PrimitiveExpr<T, L>` for primitive type \tparam T and either scalar values or vectorial ones
1457 : : * fitting in a single SIMD vector. Represents an expression (AST) evaluating to \tparam L runtime values of
1458 : : * primitive type \tparam T. */
1459 : : template<dsl_primitive T, std::size_t L>
1460 : : requires (L > 0) and (is_pow_2(L)) and (L * sizeof(T) <= 16) // since Wasm currently only supports 128 bit SIMD vectors
1461 : : struct PrimitiveExpr<T, L>
1462 : : {
1463 : : ///> the primitive type of the represented expression
1464 : : using type = T;
1465 : : ///> the number of SIMD lanes of the represented expression, i.e. 1 for scalar and at least 2 for vectorial ones
1466 : : static constexpr std::size_t num_simd_lanes = L;
1467 : :
1468 : : /*----- Friends --------------------------------------------------------------------------------------------------*/
1469 : : template<typename, std::size_t> friend struct PrimitiveExpr; // to convert U to T and U* to uint32_t
1470 : : template<typename, std::size_t>
1471 : : friend struct Expr; // to construct an empty `PrimitiveExpr<bool>` for the NULL information
1472 : : template<typename, VariableKind, bool, std::size_t>
1473 : : friend class detail::variable_storage; // to construct from `::wasm::Expression` and access private `expr()`
1474 : : friend struct Module; // to access internal `::wasm::Expression`, e.g. in `emit_return()`
1475 : : friend struct Block; // to access internal `::wasm::Expression`, e.g. in `go_to()`
1476 : : template<typename> friend struct FunctionProxy; // to access internal `::wasm::Expr` to construct function calls
1477 : : template<std::size_t> friend struct LocalBit; // to access private `move()`
1478 : : friend struct If; // to use PrimitiveExpr<bool> as condition
1479 : : friend struct While; // to use PrimitiveExpr<bool> as condition
1480 : : template<typename> friend struct invoke_interpreter; // to access private `expr()`
1481 : : template<typename, std::size_t> friend struct std::array; // to be able to default construct vectors array
1482 : :
1483 : : private:
1484 : : ///> the referenced Binaryen expression (AST)
1485 : 3252 : ::wasm::Expression *expr_ = nullptr;
1486 : : ///> a list of referenced `Bit`s
1487 : : std::list<std::shared_ptr<Bit>> referenced_bits_;
1488 : :
1489 : : private:
1490 : : ///> Constructs an empty `PrimitiveExpr`, for which `operator bool()` returns `false`.
1491 : 6504 : explicit PrimitiveExpr() = default;
1492 : :
1493 : : ///> Constructs a `PrimitiveExpr` from a Binaryen `::wasm::Expression` \p expr and the \p referenced_bits.
1494 : 152252 : explicit PrimitiveExpr(::wasm::Expression *expr, std::list<std::shared_ptr<Bit>> referenced_bits = {})
1495 : 152252 : : expr_(expr)
1496 : 152252 : , referenced_bits_(std::move(referenced_bits))
1497 : 152252 : { }
1498 : : /** Constructs a `PrimitiveExpr` from a `std::pair` of a Binaryen `::wasm::Expression` \p expr and the \p
1499 : : * referenced_bits. */
1500 : 1664 : explicit PrimitiveExpr(std::pair<::wasm::Expression*, std::list<std::shared_ptr<Bit>>> expr)
1501 [ + - + - : 1664 : : PrimitiveExpr(std::move(expr.first), std::move(expr.second))
# # # # #
# ]
1502 : 1664 : { }
1503 : :
1504 : : /** Constructs a `PrimitiveExpr` from a byte array \p bytes. */
1505 : 12 : explicit PrimitiveExpr(const std::array<uint8_t, 16> &bytes)
1506 : : requires (L > 1)
1507 [ + - + + : 12 : : PrimitiveExpr(Module::Builder().makeConst(::wasm::Literal(bytes.data())))
# # # # #
# # # ]
1508 : 12 : { }
1509 : :
1510 : : /** Constructs a `PrimitiveExpr` from a vector array \p vectors containing a single `PrimitiveExpr`. */
1511 : 26 : explicit PrimitiveExpr(std::array<PrimitiveExpr, 1> vectors)
1512 : : requires (L > 1)
1513 : 26 : : PrimitiveExpr(vectors[0])
1514 : 26 : { }
1515 : :
1516 : : public:
1517 : : /** Constructs a new `PrimitiveExpr` from a constant \p value. */
1518 : : template<dsl_primitive... Us>
1519 : : requires (sizeof...(Us) > 0) and requires (Us... us) { make_literal<T, L>(us...); }
1520 : 14708 : explicit PrimitiveExpr(Us... value)
1521 [ + - + - : 14708 : : PrimitiveExpr(Module::Builder().makeConst(make_literal<T, L>(value...)))
# # + - +
- + - + -
+ - + - +
- + - # #
+ - + - +
- # # + -
+ - + - #
# + - + -
# # + - +
- + - # #
# # + - -
+ + - - +
+ - - + #
# + - - +
+ - - + +
- + - + -
# # + - #
# + - + -
- + + - +
- + - - +
+ - + - -
+ + - + -
+ - # # #
# # # # #
# # + - #
# ]
1522 : 14708 : { }
1523 : :
1524 : : /** Constructs a new `PrimitiveExpr` from a decayable constant \p value. */
1525 : : template<decayable... Us>
1526 : : requires (sizeof...(Us) > 0) and (dsl_primitive<std::decay_t<Us>> and ...) and
1527 : : requires (Us... us) { PrimitiveExpr(std::decay_t<Us>(us)...); }
1528 : : explicit PrimitiveExpr(Us... value)
1529 : : : PrimitiveExpr(std::decay_t<Us>(value)...)
1530 : : { }
1531 : :
1532 : : PrimitiveExpr(const PrimitiveExpr&) = delete;
1533 : : /** Constructs a new `PrimitiveExpr` by **moving** the underlying `expr_` and `referenced_bits_` of `other`
1534 : : * to `this`. */
1535 : 86089 : PrimitiveExpr(PrimitiveExpr &other)
1536 [ + - + - : 86089 : : PrimitiveExpr(std::exchange(other.expr_, nullptr), std::move(other.referenced_bits_))
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
1537 : 86089 : { /* move, not copy */ }
1538 : : /** Constructs a new `PrimitiveExpr` by **moving** the underlying `expr_` and `referenced_bits_` of `other`
1539 : : * to `this`. */
1540 : 15073 : PrimitiveExpr(PrimitiveExpr &&other)
1541 [ + - + - : 15073 : : PrimitiveExpr(std::exchange(other.expr_, nullptr), std::move(other.referenced_bits_))
+ - # # #
# # # ]
1542 : 15073 : { }
1543 : :
1544 : : private:
1545 : : /** Assigns `other` to `this`. Only necessary to assign vectors array for double pumping. XXX: use vector instead of array? */
1546 : 310 : PrimitiveExpr & operator=(PrimitiveExpr other) {
1547 : : using std::swap;
1548 : 310 : swap(this->expr_, other.expr_);
1549 : 310 : swap(this->referenced_bits_, other.referenced_bits_);
1550 : 310 : return *this;
1551 : : }
1552 : :
1553 : : public:
1554 [ + - + - : 155504 : ~PrimitiveExpr() { M_insist(not expr_, "expression must be used or explicitly discarded"); }
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # +
- + - + -
+ - + - +
- + - + -
# # + - +
- + - + -
+ - + - +
- # # + -
# # # # ]
1555 : :
1556 : : private:
1557 : : /** **Moves** the underlying Binaryen `::wasm::Expression` out of `this`. */
1558 : 50540 : ::wasm::Expression * expr() {
1559 : 50540 : M_insist(expr_, "cannot access an already moved or discarded expression of a `PrimitiveExpr`");
1560 : 50540 : return std::exchange(expr_, nullptr);
1561 : : }
1562 : : /** **Moves** the referenced bits out of `this`. */
1563 : 34628 : std::list<std::shared_ptr<Bit>> referenced_bits() { return std::move(referenced_bits_); }
1564 : : /** **Moves** the underlying Binaryen `::wasm::Expression` and the referenced bits out of `this`.
1565 : : * Must be templated to match `move()` of specialization of double pumped `PrimitiveExpr`s. */
1566 : : template<dsl_primitive U = T, std::size_t M = L>
1567 : 2082 : std::pair<::wasm::Expression*, std::list<std::shared_ptr<Bit>>> move() {
1568 : 2082 : return { expr(), referenced_bits() };
1569 : : }
1570 : :
1571 : : public:
1572 : : /** Returns `true` if this `PrimitiveExpr` actually holds a value (Binaryen AST), `false` otherwise. Can be used to
1573 : : * test whether this `PrimitiveExpr` has already been used. */
1574 : 42100 : explicit operator bool() const { return expr_ != nullptr; }
1575 : :
1576 : : /** Creates and returns a *deep copy* of `this`. */
1577 : 3626 : PrimitiveExpr clone() const {
1578 : 3626 : M_insist(expr_, "cannot clone an already moved or discarded `PrimitiveExpr`");
1579 [ + - # # : 3626 : return PrimitiveExpr(
# # # # +
- + - + -
+ - + - +
- # # # #
# # + - #
# # # + -
# # + - +
- # # # #
# # + - #
# # # # #
# # # # #
# ]
1580 : 3626 : /* expr= */ ::wasm::ExpressionManipulator::copy(expr_, Module::Get().module_),
1581 : 3626 : /* referenced_bits= */ referenced_bits_ // copy
1582 : : );
1583 : 0 : }
1584 : :
1585 : : /** Discards `this`. This is necessary to signal in our DSL that a value is *expectedly* unused (and not dead
1586 : : * code). For example, the return value of a function that was invoked because of its side effects may remain
1587 : : * unused. One **must** discard the returned value to signal that the value is expectedly left unused. */
1588 : 550 : void discard() {
1589 : 550 : M_insist(expr_, "cannot discard an already moved or discarded `PrimitiveExpr`");
1590 [ + - + - : 550 : if (expr_->is<::wasm::Call>())
# # + + +
- + - + -
+ - + - #
# # # # #
# # # # #
# + - # #
+ - # # +
- # # + -
# # # # +
- # # ]
1591 : 2 : Module::Block().list.push_back(Module::Builder().makeDrop(expr_)); // keep the function call
1592 : : #ifndef NDEBUG
1593 : 550 : expr_ = nullptr;
1594 : : #endif
1595 : 550 : referenced_bits_.clear();
1596 : 550 : }
1597 : :
1598 : :
1599 : : /*------------------------------------------------------------------------------------------------------------------
1600 : : * Operation helper
1601 : : *----------------------------------------------------------------------------------------------------------------*/
1602 : :
1603 : : private:
1604 : : /** Helper function to implement *unary* operations. Applies `::wasm::UnaryOp` \p op to `this` and returns the
1605 : : * result. */
1606 : : template<dsl_primitive ResultType, std::size_t ResultL>
1607 : 6178 : PrimitiveExpr<ResultType, ResultL> unary(::wasm::UnaryOp op) {
1608 [ + - + - : 6178 : return PrimitiveExpr<ResultType, ResultL>(
+ - # # +
- + - + -
+ - + - +
- # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # # #
# # + - #
# # # # #
# # # # +
- # # + -
+ - + - +
- # # # #
# # + - +
- + - # #
# # # # #
# # # # #
# # # # #
# + - + -
# # + - #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # #
# ]
1609 : 6178 : /* expr= */ Module::Builder().makeUnary(op, expr()),
1610 : 6178 : /* referenced_bits= */ referenced_bits() // moved
1611 : : );
1612 : 0 : }
1613 : :
1614 : : /** Helper function to implement *binary* operations. Applies `::wasm::BinaryOp` \p op to `this` and \p other and
1615 : : * returns the result. Note, that we require `this` and \p other to be of same type. */
1616 : : template<dsl_primitive ResultType, std::size_t ResultL, dsl_primitive OperandType, std::size_t OperandL>
1617 : 12196 : PrimitiveExpr<ResultType, ResultL> binary(::wasm::BinaryOp op, PrimitiveExpr<OperandType, OperandL> other) {
1618 : 12196 : auto referenced_bits = this->referenced_bits(); // moved
1619 [ + - + - : 12196 : referenced_bits.splice(referenced_bits.end(), other.referenced_bits());
+ - + - +
- + - # #
+ - + - #
# # # # #
+ - # # +
- # # # #
# # ]
1620 [ - + - + : 12196 : return PrimitiveExpr<ResultType, ResultL>(
- + - + -
+ - + - +
- + # # -
+ - + - +
- + # # -
+ # # - +
- + - + -
+ # # # #
- + - + -
+ - + # #
# # - + #
# # # - +
# # # # -
+ - + - +
# # - + -
+ # # - +
# # - + #
# # # # #
# # # # #
# # # - +
- + # # #
# - + - +
# # # # -
+ # # - +
# # # # #
# # # # #
# # - + #
# # # #
# ]
1621 [ + - + - : 24392 : /* expr= */ Module::Builder().makeBinary(op,
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
# # + - +
- + - + -
+ - + - +
- + - # #
# # + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - # # #
# # # # #
+ - + - #
# # # # #
# # + - +
- + - + -
+ - + - #
# # # + -
+ - + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - +
- + - # #
# # # # #
# + - + -
+ - + - #
# # # # #
# # + - +
- # # # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - # # #
# # # # #
# # # # ]
1622 [ + - + - : 12196 : this->template to<OperandType, OperandL>().expr(),
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
# # + - +
- + - + -
+ - + - +
- + - # #
# # + - +
- # # # #
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - # # #
# # # # #
+ - + - #
# # # # #
# # + - +
- + - + -
+ - + - #
# # # + -
+ - + - +
- # # # #
+ - + - #
# # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - +
- + - # #
# # # # #
# + - + -
+ - + - #
# # # # #
# # + - +
- # # # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - # # #
# # # # #
# # # # ]
1623 [ + - + - : 12196 : other.expr()),
+ - + - +
- + - + -
+ - # # +
- + - + -
+ - # # +
- # # + -
+ - + - +
- # # # #
+ - + - +
- + - # #
# # + - #
# # # + -
# # # # +
- + - + -
# # + - +
- # # + -
# # + - #
# # # # #
# # # # #
# # # + -
+ - # # #
# + - + -
# # # # +
- # # + -
# # # # #
# # # # #
# # + - #
# # # #
# ]
1624 : 12196 : /* referenced_bits= */ std::move(referenced_bits)
1625 : : );
1626 : 12196 : }
1627 : :
1628 : :
1629 : : /*------------------------------------------------------------------------------------------------------------------
1630 : : * Conversion operations
1631 : : *----------------------------------------------------------------------------------------------------------------*/
1632 : :
1633 : : private:
1634 : : template<dsl_primitive U, std::size_t M>
1635 : 15080 : PrimitiveExpr<U, M> convert() {
1636 : : using From = T;
1637 : : using To = U;
1638 : 15080 : constexpr std::size_t FromL = L;
1639 : 15080 : constexpr std::size_t ToL = M;
1640 : :
1641 : : if constexpr (std::same_as<From, To> and FromL == ToL)
1642 : 12754 : return *this;
1643 : : if constexpr (integral<From> and integral<To> and std::is_signed_v<From> == std::is_signed_v<To> and
1644 : : sizeof(From) == sizeof(To) and FromL == ToL)
1645 : : return PrimitiveExpr<To, ToL>(move());
1646 : :
1647 : : if constexpr (boolean<From>) { // from boolean
1648 : : if constexpr (integral<To>) { // to integer
1649 : : if constexpr (FromL == 1 and ToL == 1) { // scalar
1650 : : if constexpr (sizeof(To) <= 4) // bool -> i32
1651 [ # # + - : 186 : return PrimitiveExpr<To, ToL>(move());
# # # # +
- ]
1652 : : if constexpr (sizeof(To) == 8) // bool -> i64
1653 : 624 : return unary<To, ToL>(::wasm::ExtendUInt32);
1654 : : }
1655 : : if constexpr (FromL > 1 and FromL == ToL) { // vectorial
1656 : : if constexpr (sizeof(To) == 1) // bool -> i8/u8
1657 [ + - - + : 34 : return -PrimitiveExpr<To, ToL>(move()); // negate to convert 0xff to 1
# # # # #
# # # # #
# # + - -
+ ]
1658 : : if constexpr (std::is_signed_v<To>) {
1659 : : if constexpr (sizeof(To) == 2) // bool -> i16
1660 [ # # ]: 0 : return to<int8_t, ToL>().template to<To, ToL>();
1661 : : if constexpr (sizeof(To) == 4) // bool -> i32
1662 [ # # ]: 0 : return to<int8_t, ToL>().template to<To, ToL>();
1663 : : if constexpr (sizeof(To) == 8) // bool -> i64
1664 [ # # # # : 0 : return to<int8_t, ToL>().template to<To, ToL>();
# # # # ]
1665 : : } else {
1666 : : if constexpr (sizeof(To) == 2) // bool -> u16
1667 [ # # # # : 0 : return to<uint8_t, ToL>().template to<To, ToL>();
# # # # ]
1668 : : if constexpr (sizeof(To) == 4) // bool -> u32
1669 [ # # # # : 0 : return to<uint8_t, ToL>().template to<To, ToL>();
# # # # ]
1670 : : if constexpr (sizeof(To) == 8) // bool -> u64
1671 [ # # # # : 0 : return to<uint8_t, ToL>().template to<To, ToL>();
# # # # ]
1672 : : }
1673 : : }
1674 : : }
1675 : : if constexpr (std::floating_point<To>) { // to floating point
1676 : : if constexpr (FromL == 1 and ToL == 1) { // scalar
1677 : : if constexpr (sizeof(To) == 4) // bool -> f32
1678 : 0 : return unary<To, ToL>(::wasm::ConvertUInt32ToFloat32);
1679 : : if constexpr (sizeof(To) == 8) // bool -> f64
1680 : 0 : return unary<To, ToL>(::wasm::ConvertUInt32ToFloat64);
1681 : : }
1682 : : if constexpr (FromL > 1 and FromL == ToL) { // vectorial
1683 : : if constexpr (sizeof(To) == 4) // bool -> f32
1684 [ # # ]: 0 : return to<uint32_t, ToL>().template convert<To, ToL>();
1685 : : if constexpr (sizeof(To) == 8) // bool -> f64
1686 [ # # ]: 0 : return to<uint32_t, ToL>().template convert<To, ToL>();
1687 : : }
1688 : : }
1689 : : }
1690 : :
1691 : : if constexpr (boolean<To>) // to boolean
1692 [ + - + - : 758 : return *this != PrimitiveExpr(static_cast<From>(0));
# # + - +
- + - + -
# # # # #
# # # ]
1693 : :
1694 : : if constexpr (integral<From>) { // from integer
1695 : : if constexpr (integral<To>) { // to integer
1696 : : if constexpr (FromL == 1 and ToL == 1) { // scalar
1697 : : if constexpr (std::is_signed_v<From>) { // signed
1698 : : if constexpr (sizeof(From) <= 4 and sizeof(To) == 8) // i32 -> i64
1699 : 0 : return unary<To, ToL>(::wasm::ExtendSInt32);
1700 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // i64 -> i32
1701 : 54 : return unary<To, ToL>(::wasm::WrapInt64);
1702 : : } else { // unsigned
1703 : : if constexpr (sizeof(From) <= 4 and sizeof(To) == 8) // u32 -> u64
1704 : 300 : return unary<To, ToL>(::wasm::ExtendUInt32);
1705 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // u64 -> u32
1706 : 0 : return unary<To, ToL>(::wasm::WrapInt64);
1707 : : }
1708 : : if constexpr (sizeof(To) <= 4 and sizeof(From) < sizeof(To)) // From less precise than To
1709 [ + - + - : 242 : return PrimitiveExpr<To, ToL>(move()); // extend integer
+ - + - #
# # # ]
1710 : : if constexpr (sizeof(From) <= 4 and sizeof(To) < sizeof(From)) { // To less precise than From
1711 : 20 : constexpr From MASK = (uint64_t(1) << (8 * sizeof(To))) - uint64_t(1);
1712 [ # # # # : 20 : return PrimitiveExpr<To, ToL>((*this bitand PrimitiveExpr(MASK)).move()); // truncate integer
# # + - +
- - + # #
# # # # +
- + - - +
# # # # #
# ]
1713 : : }
1714 : : if constexpr (sizeof(From) == 8 and sizeof(To) < 4) { // To less precise than From
1715 : : if constexpr (std::is_signed_v<To>) {
1716 : 0 : auto wrapped = unary<int32_t, ToL>(::wasm::WrapInt64); // wrap integer
1717 : 0 : constexpr int32_t MASK = (int64_t(1) << (8 * sizeof(To))) - int64_t(1);
1718 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(
1719 [ # # # # : 0 : (wrapped bitand PrimitiveExpr<int32_t, ToL>(MASK)).move() // truncate integer
# # # # #
# # # ]
1720 : : );
1721 : 0 : } else {
1722 : 42 : auto wrapped = unary<uint32_t, ToL>(::wasm::WrapInt64); // wrap integer
1723 : 42 : constexpr uint32_t MASK = (uint64_t(1) << (8 * sizeof(To))) - uint64_t(1);
1724 [ - + ]: 42 : return PrimitiveExpr<To, ToL>(
1725 [ + - + - : 42 : (wrapped bitand PrimitiveExpr<uint32_t, ToL>(MASK)).move() // truncate integer
+ - ]
1726 : : );
1727 : 42 : }
1728 : : }
1729 : : }
1730 : : if constexpr (FromL > 1 and FromL == ToL) { // vectorial
1731 : : if constexpr (std::is_signed_v<From>) { // signed
1732 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 2 and FromL <= 8) // i8 -> i16
1733 : 10 : return unary<To, ToL>(::wasm::ExtendLowSVecI8x16ToVecI16x8);
1734 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 2 and FromL == 16) { // i8 -> i16
1735 : 0 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1736 : 0 : vectors[0] =
1737 [ # # # # ]: 0 : clone().template unary<To, ToL / 2>(::wasm::ExtendLowSVecI8x16ToVecI16x8);
1738 [ # # ]: 0 : vectors[1] = unary<To, ToL / 2>(::wasm::ExtendHighSVecI8x16ToVecI16x8);
1739 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1740 : 0 : }
1741 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // i8 -> i32
1742 [ + - + - : 10 : return to<int16_t, ToL>().template to<To, ToL>();
# # # # ]
1743 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // i8 -> i64
1744 [ # # + - : 4 : return to<int32_t, ToL>().template to<To, ToL>();
# # # # ]
1745 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4 and FromL <= 4) // i16 -> i32
1746 : 10 : return unary<To, ToL>(::wasm::ExtendLowSVecI16x8ToVecI32x4);
1747 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4 and FromL == 8) { // i16 -> i32
1748 : 0 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1749 : 0 : vectors[0] =
1750 [ # # # # ]: 0 : clone().template unary<To, ToL / 2>(::wasm::ExtendLowSVecI16x8ToVecI32x4);
1751 [ # # ]: 0 : vectors[1] = unary<To, ToL / 2>(::wasm::ExtendHighSVecI16x8ToVecI32x4);
1752 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1753 : 0 : }
1754 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // i16 -> i64
1755 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
1756 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL <= 2) // i32 -> i64
1757 : 0 : return unary<To, ToL>(::wasm::ExtendLowSVecI32x4ToVecI64x2);
1758 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL == 4) { // i32 -> i64
1759 : 4 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1760 : 4 : vectors[0] =
1761 [ + - + - ]: 4 : clone().template unary<To, ToL / 2>(::wasm::ExtendLowSVecI32x4ToVecI64x2);
1762 [ + - ]: 4 : vectors[1] = unary<To, ToL / 2>(::wasm::ExtendHighSVecI32x4ToVecI64x2);
1763 [ + - - + ]: 4 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1764 : 4 : }
1765 : : } else { // unsigned
1766 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 2 and FromL <= 8) // u8 -> u16
1767 : 4 : return unary<To, ToL>(::wasm::ExtendLowUVecI8x16ToVecI16x8);
1768 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 2 and FromL == 16) { // u8 -> u16
1769 : 0 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1770 : 0 : vectors[0] =
1771 [ # # # # ]: 0 : clone().template unary<To, ToL / 2>(::wasm::ExtendLowUVecI8x16ToVecI16x8);
1772 [ # # ]: 0 : vectors[1] = unary<To, ToL / 2>(::wasm::ExtendHighUVecI8x16ToVecI16x8);
1773 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1774 : 0 : }
1775 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // u8 -> u32
1776 [ # # + - : 4 : return to<uint16_t, ToL>().template to<To, ToL>();
# # # # ]
1777 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // u8 -> u64
1778 [ # # # # : 0 : return to<uint32_t, ToL>().template to<To, ToL>();
# # # # ]
1779 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4 and FromL <= 4) // u16 -> u32
1780 : 4 : return unary<To, ToL>(::wasm::ExtendLowUVecI16x8ToVecI32x4);
1781 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4 and FromL == 8) { // u16 -> u32
1782 : 0 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1783 : 0 : vectors[0] =
1784 [ # # # # ]: 0 : clone().template unary<To, ToL / 2>(::wasm::ExtendLowUVecI16x8ToVecI32x4);
1785 [ # # ]: 0 : vectors[1] = unary<To, ToL / 2>(::wasm::ExtendHighUVecI16x8ToVecI32x4);
1786 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1787 : 0 : }
1788 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // u16 -> u64
1789 : : return to<uint32_t, ToL>().template to<To, ToL>();
1790 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL <= 2) // u32 -> u64
1791 : 0 : return unary<To, ToL>(::wasm::ExtendLowUVecI32x4ToVecI64x2);
1792 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL == 4) { // u32 -> u64
1793 : 0 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1794 : 0 : vectors[0] =
1795 [ # # # # ]: 0 : clone().template unary<To, ToL / 2>(::wasm::ExtendLowUVecI32x4ToVecI64x2);
1796 [ # # ]: 0 : vectors[1] = unary<To, ToL / 2>(::wasm::ExtendHighUVecI32x4ToVecI64x2);
1797 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1798 : 0 : }
1799 : : }
1800 : : }
1801 : : }
1802 : : if constexpr (std::floating_point<To>) { // to floating point
1803 : : if constexpr (FromL == 1 and ToL == 1) { // scalar
1804 : : if constexpr (std::is_signed_v<From>) { // signed
1805 : : if constexpr (sizeof(From) <= 4 and sizeof(To) == 4) // i32 -> f32
1806 : 0 : return unary<To, ToL>(::wasm::ConvertSInt32ToFloat32);
1807 : : if constexpr (sizeof(From) <= 4 and sizeof(To) == 8) // i32 -> f64
1808 : 0 : return unary<To, ToL>(::wasm::ConvertSInt32ToFloat64);
1809 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // i64 -> f32
1810 : 0 : return unary<To, ToL>(::wasm::ConvertSInt64ToFloat32);
1811 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // i64 -> f64
1812 : 0 : return unary<To, ToL>(::wasm::ConvertSInt64ToFloat64);
1813 : : } else { // unsigned
1814 : : if constexpr (sizeof(From) <= 4 and sizeof(To) == 4) // u32 -> f32
1815 : : return unary<To, ToL>(::wasm::ConvertUInt32ToFloat32);
1816 : : if constexpr (sizeof(From) <= 4 and sizeof(To) == 8) // u32 -> f64
1817 : : return unary<To, ToL>(::wasm::ConvertUInt32ToFloat64);
1818 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // u64 -> f32
1819 : : return unary<To, ToL>(::wasm::ConvertUInt64ToFloat32);
1820 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // u64 -> f64
1821 : : return unary<To, ToL>(::wasm::ConvertUInt64ToFloat64);
1822 : : }
1823 : : }
1824 : : if constexpr (FromL > 1 and FromL == ToL) { // vectorial
1825 : : if constexpr (std::is_signed_v<From>) { // signed
1826 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // i8 -> f32
1827 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
1828 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // i8 -> f64
1829 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
1830 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4) // i16 -> f32
1831 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
1832 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // i16 -> f64
1833 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
1834 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) // i32 -> f32
1835 : 0 : return unary<To, ToL>(::wasm::ConvertSVecI32x4ToVecF32x4);
1836 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL <= 2) // i32 -> f64
1837 : : return unary<To, ToL>(::wasm::ConvertLowSVecI32x4ToVecF64x2);
1838 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL == 4) { // i32 -> f64
1839 : 0 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1840 : 0 : vectors[0] =
1841 [ # # # # ]: 0 : clone().template unary<To, ToL / 2>(::wasm::ConvertLowSVecI32x4ToVecF64x2);
1842 [ # # ]: 0 : auto high_to_low = swizzle_lanes(std::to_array<uint8_t>({ 2, 3 }));
1843 : 0 : vectors[1] =
1844 [ # # ]: 0 : high_to_low.template unary<To, ToL / 2>(::wasm::ConvertLowSVecI32x4ToVecF64x2);
1845 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1846 : 0 : }
1847 : : } else { // unsigned
1848 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // u8 -> f32
1849 : : return to<uint32_t, ToL>().template convert<To, ToL>();
1850 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // u8 -> f64
1851 : : return to<uint32_t, ToL>().template convert<To, ToL>();
1852 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4) // u16 -> f32
1853 : : return to<uint32_t, ToL>().template convert<To, ToL>();
1854 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // u16 -> f64
1855 : : return to<uint32_t, ToL>().template convert<To, ToL>();
1856 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) // u32 -> f32
1857 : : return unary<To, ToL>(::wasm::ConvertUVecI32x4ToVecF32x4);
1858 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL <= 2) // u32 -> f64
1859 : : return unary<To, ToL>(::wasm::ConvertLowUVecI32x4ToVecF64x2);
1860 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL == 4) { // u32 -> f64
1861 : : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1862 : : vectors[0] =
1863 : : clone().template unary<To, ToL / 2>(::wasm::ConvertLowUVecI32x4ToVecF64x2);
1864 : : auto high_to_low = swizzle_lanes(std::to_array<uint8_t>({ 2, 3 }));
1865 : : vectors[1] =
1866 : : high_to_low.template unary<To, ToL / 2>(::wasm::ConvertLowUVecI32x4ToVecF64x2);
1867 : : return PrimitiveExpr<To, ToL>(std::move(vectors));
1868 : : }
1869 : : }
1870 : : }
1871 : : }
1872 : : }
1873 : :
1874 : : if constexpr (std::floating_point<From>) { // from floating point
1875 : : if constexpr (integral<To>) { // to integer
1876 : : if constexpr (FromL == 1 and ToL == 1) { // scalar
1877 : : if constexpr (std::is_signed_v<To>) { // signed
1878 : : if constexpr (sizeof(From) == 4 and sizeof(To) <= 4) // f32 -> i32
1879 [ # # # # : 0 : return unary<int32_t, ToL>(::wasm::TruncSFloat32ToInt32).template to<To, ToL>();
# # ]
1880 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) // f32 -> i64
1881 : 0 : return unary<To, ToL>(::wasm::TruncSFloat32ToInt64);
1882 : : if constexpr (sizeof(From) == 8 and sizeof(To) <= 4) // f64 -> i32
1883 [ # # # # : 0 : return unary<int32_t, ToL>(::wasm::TruncSFloat64ToInt32).template to<To, ToL>();
# # ]
1884 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // f64 -> i64
1885 : 0 : return unary<To, ToL>(::wasm::TruncSFloat64ToInt64);
1886 : : } else { // unsigned
1887 : : if constexpr (sizeof(From) == 4 and sizeof(To) <= 4) // f32 -> u32
1888 : : return unary<uint32_t, ToL>(::wasm::TruncUFloat32ToInt32).template to<To, ToL>();
1889 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) // f32 -> u64
1890 : : return unary<To, ToL>(::wasm::TruncUFloat32ToInt64);
1891 : : if constexpr (sizeof(From) == 8 and sizeof(To) <= 4) // f64 -> u32
1892 : : return unary<uint32_t, ToL>(::wasm::TruncUFloat64ToInt32).template to<To, ToL>();
1893 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // f64 -> u64
1894 : : return unary<To, ToL>(::wasm::TruncUFloat64ToInt64);
1895 : : }
1896 : : }
1897 : : if constexpr (FromL > 1 and FromL == ToL) { // vectorial
1898 : : if constexpr (std::is_signed_v<To>) { // signed
1899 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) // f32 -> i32
1900 : 0 : return unary<To, ToL>(::wasm::TruncSatSVecF32x4ToVecI32x4);
1901 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // f64 -> i32
1902 : 0 : return unary<To, ToL>(::wasm::TruncSatZeroSVecF64x2ToVecI32x4);
1903 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) // f32 -> i64
1904 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
1905 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // f64 -> i64
1906 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
1907 : : } else { // unsigned
1908 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) // f32 -> u32
1909 : : return unary<To, ToL>(::wasm::TruncSatUVecF32x4ToVecI32x4);
1910 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // f64 -> u32
1911 : : return unary<To, ToL>(::wasm::TruncSatZeroUVecF64x2ToVecI32x4);
1912 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) // f32 -> u64
1913 : : return convert<uint32_t, ToL>().template to<To, ToL>();
1914 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // f64 -> u64
1915 : : return convert<uint32_t, ToL>().template to<To, ToL>();
1916 : : }
1917 : : }
1918 : : }
1919 : : if constexpr (std::floating_point<To>) { // to floating point
1920 : : if constexpr (FromL == 1 and ToL == 1) { // scalar
1921 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) // f32 -> f64
1922 : 12 : return unary<To, ToL>(::wasm::PromoteFloat32);
1923 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // f64 -> f32
1924 : 4 : return unary<To, ToL>(::wasm::DemoteFloat64);
1925 : : }
1926 : : if constexpr (FromL > 1 and FromL == ToL) { // vectorial
1927 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL <= 2) // f32 -> f64
1928 : : return unary<To, ToL>(::wasm::PromoteLowVecF32x4ToVecF64x2);
1929 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8 and FromL == 4) { // f32 -> f64
1930 : 0 : std::array<PrimitiveExpr<To, ToL / 2>, 2> vectors;
1931 [ # # # # ]: 0 : vectors[0] = clone().template unary<To, ToL / 2>(::wasm::PromoteLowVecF32x4ToVecF64x2);
1932 [ # # ]: 0 : auto high_to_low = swizzle_lanes(std::to_array<uint8_t>({ 2, 3 }));
1933 : 0 : vectors[1] =
1934 [ # # ]: 0 : high_to_low.template unary<To, ToL / 2>(::wasm::PromoteLowVecF32x4ToVecF64x2);
1935 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
1936 : 0 : }
1937 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // f64 -> f32
1938 : : return unary<To, ToL>(::wasm::DemoteZeroVecF64x2ToVecF32x4);
1939 : : }
1940 : : }
1941 : : }
1942 : :
1943 : : M_unreachable("illegal conversion");
1944 : 46 : }
1945 : :
1946 : : public:
1947 : : /** Implicit conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<To, ToL>`. Only applicable if
1948 : : *
1949 : : * - `L` and `ToL` are equal, i.e. conversion does not change the number of SIMD lanes
1950 : : * - `T` and `To` have same signedness
1951 : : * - neither or both `T` and `To` are integers
1952 : : * - `T` can be *trivially* converted to `To` (e.g. `int` to `long` but not `long` to `int`)
1953 : : * - `To` is not `bool`
1954 : : */
1955 : : template<dsl_primitive To, std::size_t ToL = L>
1956 : : requires (L == ToL) and // L and ToL are equal
1957 : : same_signedness<T, To> and // T and To have same signedness
1958 : : (integral<T> == integral<To>) and // neither nor both T and To are integers (excluding bool)
1959 : : (sizeof(T) <= sizeof(To)) // T can be *trivially* converted to To
1960 : 332 : operator PrimitiveExpr<To, ToL>() { return convert<To, ToL>(); }
1961 : :
1962 : : /** Explicit conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<To, ToL>`. Only applicable if
1963 : : *
1964 : : * - `L` and `ToL` are equal, i.e. conversion does not change the number of SIMD lanes
1965 : : * - `T` and `To` have same signedness or `T` is `bool` or `char` or `To` is `bool` or `char`
1966 : : * - for scalar values:
1967 : : * - `T` can be converted to `To` (e.g. `int` to `long`, `long` to `int`, `float` to `int`)
1968 : : * - for vectorial values:
1969 : : * - `T` can be converted to `To` (e.g. `int` to `long`, `float` to `int`) except integer conversion to a less
1970 : : * precise type, conversion from 64-bit integer to floating point, or conversion from floating point to 8-bit
1971 : : * or 16-bit integer
1972 : : */
1973 : : template<dsl_primitive To, std::size_t ToL = L>
1974 : : requires (L == ToL) and // L and ToL are equal
1975 : : (same_signedness<T, To> or // T and To have same signedness
1976 : : boolean<T> or std::same_as<T, char> or // or T is bool or char
1977 : : boolean<To> or std::same_as<To, char>) and // or To is bool or char
1978 : : ((L != 1) or // for scalar values
1979 : : std::is_convertible_v<T, To>) and // T can be converted to To
1980 : : ((L == 1) or // for vectorial values
1981 : : (std::is_convertible_v<T, To> and // T can be converted to To
1982 : : not (integral<T> and integral<To> and sizeof(T) > sizeof(To)) and // except integer conversion to less precise type
1983 : : not (integral<T> and sizeof(T) == 8 and std::floating_point<To>) and // except 64-bit integer to floating point
1984 : : not (std::floating_point<T> and integral<To> and sizeof(To) <= 2))) // except floating point to 8-bit or 16-bit integer
1985 : 14748 : PrimitiveExpr<To, ToL> to() { return convert<To, ToL>(); }
1986 : :
1987 : : /** Explicit conversion of a `PrimitiveExpr<uint32_t, 1>` to a `PrimitiveExpr<To*, ToL>`
1988 : : *
1989 : : * - `T` is `uint32_t`
1990 : : * - `L` equals 1, i.e. its scalar
1991 : : * - `To` is a pointer to primitive type
1992 : : */
1993 : : template<dsl_pointer_to_primitive To, std::size_t ToL = L>
1994 : 2456 : PrimitiveExpr<To, ToL> to() requires std::same_as<T, uint32_t> and (L == 1) {
1995 [ + - + - : 2456 : return PrimitiveExpr<To, ToL>(*this);
# # + - +
- + - #
# ]
1996 : 0 : }
1997 : :
1998 : : /** Conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<std::make_signed_t<T>, L>`. Only applicable if
1999 : : *
2000 : : * - `T` is an unsigned integral type except `bool`
2001 : : */
2002 [ + - + - ]: 178 : auto make_signed() requires unsigned_integral<T> { return PrimitiveExpr<std::make_signed_t<T>, L>(move()); }
2003 : :
2004 : : /** Conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<std::make_unsigned_t<T>, L>`. Only available if
2005 : : *
2006 : : * - `T` is a signed integral type except `bool`
2007 : : */
2008 [ + - # # : 858 : auto make_unsigned() requires signed_integral<T> { return PrimitiveExpr<std::make_unsigned_t<T>, L>(move()); }
# # # # ]
2009 : :
2010 : : /** Reinterpretation of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<To, L>`. Only applicable if
2011 : : *
2012 : : * - `L` equals 1, i.e. the value is scalar
2013 : : * - `T` is integral and `To` is floating point or vice versa
2014 : : * - `T` and `To` have same size
2015 : : */
2016 : : template<dsl_primitive To, std::size_t ToL = L>
2017 : : requires (L == ToL) and (L == 1) and // value is scalar
2018 : : ((integral<T> and std::floating_point<To>) or // either T is integral and To is floating point
2019 : : (std::floating_point<T> and integral<To>)) and // or vice versa
2020 : : (sizeof(T) == sizeof(To)) // T and To have same size
2021 : 0 : PrimitiveExpr<To, ToL> reinterpret() {
2022 : : using From = T;
2023 : 0 : constexpr std::size_t FromL = L;
2024 : :
2025 : : if constexpr (integral<From>) { // from integer
2026 : : if constexpr (std::floating_point<To>) { // to floating point
2027 : : if constexpr (FromL == 1) { // scalar
2028 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) // i32 -> f32
2029 : : return unary<To, ToL>(::wasm::ReinterpretInt32);
2030 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // i64 -> f64
2031 : : return unary<To, ToL>(::wasm::ReinterpretInt64);
2032 : : }
2033 : : }
2034 : : }
2035 : :
2036 : : if constexpr (std::floating_point<From>) { // from floating point
2037 : : if constexpr (integral<To>) { // to integer
2038 : : if constexpr (FromL == 1) { // scalar
2039 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) // f32 -> i32
2040 : 0 : return unary<To, ToL>(::wasm::ReinterpretFloat32);
2041 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // f64 -> i64
2042 : 0 : return unary<To, ToL>(::wasm::ReinterpretFloat64);
2043 : : }
2044 : : }
2045 : : }
2046 : :
2047 : : M_unreachable("illegal reinterpretation");
2048 : : }
2049 : :
2050 : : /** Broadcasts a `PrimitiveExpr<T, 1>` to a `PrimitiveExpr<T, ToL>`. */
2051 : : template<std::size_t ToL>
2052 : : requires (L == 1) and (ToL > 1) and (ToL * sizeof(T) <= 16)
2053 : 0 : PrimitiveExpr<T, ToL> broadcast() {
2054 : : if constexpr (boolean<T>)
2055 : 0 : return unary<T, ToL>(::wasm::UnaryOp::SplatVecI8x16);
2056 : :
2057 : : if constexpr (integral<T>) {
2058 : : if constexpr (sizeof(T) == 1)
2059 : : return unary<T, ToL>(::wasm::UnaryOp::SplatVecI8x16);
2060 : : if constexpr (sizeof(T) == 2)
2061 : 0 : return unary<T, ToL>(::wasm::UnaryOp::SplatVecI16x8);
2062 : : if constexpr (sizeof(T) == 4)
2063 : 0 : return unary<T, ToL>(::wasm::UnaryOp::SplatVecI32x4);
2064 : : if constexpr (sizeof(T) == 8)
2065 : : return unary<T, ToL>(::wasm::UnaryOp::SplatVecI64x2);
2066 : : }
2067 : :
2068 : : if constexpr (std::floating_point<T>) {
2069 : : if constexpr (sizeof(T) == 4)
2070 : : return unary<T, ToL>(::wasm::UnaryOp::SplatVecF32x4);
2071 : : if constexpr (sizeof(T) == 8)
2072 : : return unary<T, ToL>(::wasm::UnaryOp::SplatVecF64x2);
2073 : : }
2074 : :
2075 : : M_unreachable("illegal broadcast");
2076 : : }
2077 : : /** Broadcasts a `PrimitiveExpr<T, 1>` to a `PrimitiveExpr<T, ToL>`. */
2078 : : template<std::size_t ToL>
2079 : : requires (L == 1) and (ToL > 1) and (ToL * sizeof(T) > 16)
2080 : 0 : PrimitiveExpr<T, ToL> broadcast() {
2081 : : using ResT = PrimitiveExpr<T, ToL>;
2082 : 0 : std::array<typename ResT::vector_type, ResT::num_vectors> vectors;
2083 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < ResT::num_vectors; ++idx)
2084 [ # # # # : 0 : vectors[idx] = clone().template broadcast<ResT::vector_type::num_simd_lanes>();
# # # # ]
2085 [ # # # # : 0 : return ResT(std::move(vectors));
# # # # ]
2086 : 0 : }
2087 : :
2088 : :
2089 : : /*------------------------------------------------------------------------------------------------------------------
2090 : : * Unary operations
2091 : : *----------------------------------------------------------------------------------------------------------------*/
2092 : :
2093 : : #define UNOP_(NAME, TYPE) (::wasm::UnaryOp::NAME##TYPE)
2094 : : #define UNIOP_(NAME) [] { \
2095 : : if constexpr (sizeof(T) == 8) \
2096 : : return UNOP_(NAME,Int64); \
2097 : : else if constexpr (sizeof(T) <= 4) \
2098 : : return UNOP_(NAME,Int32); \
2099 : : else \
2100 : : M_unreachable("unsupported operation"); \
2101 : : } ()
2102 : : #define UNFOP_(NAME) [] { \
2103 : : if constexpr (sizeof(T) == 8) \
2104 : : return UNOP_(NAME,Float64); \
2105 : : else if constexpr (sizeof(T) == 4) \
2106 : : return UNOP_(NAME,Float32); \
2107 : : else \
2108 : : M_unreachable("unsupported operation"); \
2109 : : } ()
2110 : : #define UNVOP_(NAME, TYPE) (::wasm::UnaryOp::NAME##Vec##TYPE)
2111 : : #define UNIVOP_(NAME) [] { \
2112 : : if constexpr (sizeof(T) == 8) \
2113 : : return UNVOP_(NAME,I64x2); \
2114 : : else if constexpr (sizeof(T) == 4) \
2115 : : return UNVOP_(NAME,I32x4); \
2116 : : else if constexpr (sizeof(T) == 2) \
2117 : : return UNVOP_(NAME,I16x8); \
2118 : : else if constexpr (sizeof(T) == 1) \
2119 : : return UNVOP_(NAME,I8x16); \
2120 : : else \
2121 : : M_unreachable("unsupported operation"); \
2122 : : } ()
2123 : : #define UNFVOP_(NAME) [] { \
2124 : : if constexpr (sizeof(T) == 8) \
2125 : : return UNVOP_(NAME,F64x2); \
2126 : : else if constexpr (sizeof(T) == 4) \
2127 : : return UNVOP_(NAME,F32x4); \
2128 : : else \
2129 : : M_unreachable("unsupported operation"); \
2130 : : } ()
2131 : : #define UNARY_VOP(NAME) [] { \
2132 : : if constexpr (std::integral<T>) \
2133 : : return UNIVOP_(NAME); \
2134 : : else if constexpr (std::floating_point<T>) \
2135 : : return UNFVOP_(NAME); \
2136 : : else \
2137 : : M_unreachable("unsupported operation"); \
2138 : : } ()
2139 : :
2140 : : /*----- Arithmetical operations ----------------------------------------------------------------------------------*/
2141 : :
2142 : 12 : PrimitiveExpr operator+() requires arithmetic<T> { return *this; }
2143 : :
2144 [ + - - + : 26 : PrimitiveExpr operator-() requires integral<T> and (L == 1) { return PrimitiveExpr(T(0)) - *this; }
# # # # +
- - + # #
# # # # #
# ]
2145 : 8 : PrimitiveExpr operator-() requires std::floating_point<T> and (L == 1) { return unary<T, L>(UNFOP_(Neg)); }
2146 : 114 : PrimitiveExpr operator-() requires arithmetic<T> and (L > 1) { return unary<T, L>(UNARY_VOP(Neg)); }
2147 : :
2148 : 16 : PrimitiveExpr abs() requires std::floating_point<T> and (L == 1) { return unary<T, L>(UNFOP_(Abs)); }
2149 : 12 : PrimitiveExpr abs() requires (L > 1) { return unary<T, L>(UNARY_VOP(Abs)); }
2150 : 8 : PrimitiveExpr ceil() requires std::floating_point<T> and (L == 1) { return unary<T, L>(UNFOP_(Ceil)); }
2151 : 4 : PrimitiveExpr ceil() requires std::floating_point<T> and (L > 1) { return unary<T, L>(UNFVOP_(Ceil)); }
2152 : 8 : PrimitiveExpr floor() requires std::floating_point<T> and (L == 1) { return unary<T, L>(UNFOP_(Floor)); }
2153 : 4 : PrimitiveExpr floor() requires std::floating_point<T> and (L > 1) { return unary<T, L>(UNFVOP_(Floor)); }
2154 : 4 : PrimitiveExpr trunc() requires std::floating_point<T> and (L > 1) { return unary<T, L>(UNFVOP_(Trunc)); }
2155 : 4 : PrimitiveExpr nearest() requires std::floating_point<T> and (L > 1) { return unary<T, L>(UNFVOP_(Nearest)); }
2156 : :
2157 : 8 : PrimitiveExpr sqrt() requires std::floating_point<T> and (L == 1) { return unary<T, L>(UNFOP_(Sqrt)); }
2158 : 4 : PrimitiveExpr sqrt() requires std::floating_point<T> and (L > 1) { return unary<T, L>(UNFVOP_(Sqrt)); }
2159 : :
2160 : 6 : PrimitiveExpr<int16_t, L / 2> add_pairwise() requires signed_integral<T> and (sizeof(T) == 1) and (L > 1) {
2161 : : static_assert(L % 2 == 0, "must mask this expression first");
2162 : 6 : auto vec = unary<int16_t, L / 2>(UNVOP_(ExtAddPairwiseS, I8x16ToI16x8));
2163 [ + - + - : 12 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
+ - ]
2164 : : vec.template extract_unsafe<0>(), // extract a single sum from vector to scalar
2165 : : vec);
2166 : 6 : }
2167 : 6 : PrimitiveExpr<uint16_t, L / 2> add_pairwise() requires unsigned_integral<T> and (sizeof(T) == 1) and (L > 1) {
2168 : : static_assert(L % 2 == 0, "must mask this expression first");
2169 : 6 : auto vec = unary<uint16_t, L / 2>(UNVOP_(ExtAddPairwiseU, I8x16ToI16x8));
2170 [ # # + - : 12 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
+ - ]
2171 : : vec.template extract_unsafe<0>(), // extract a single sum from vector to scalar
2172 : : vec);
2173 : 6 : }
2174 : 4 : PrimitiveExpr<int32_t, L / 2> add_pairwise() requires signed_integral<T> and (sizeof(T) == 2) and (L > 1) {
2175 : : static_assert(L % 2 == 0, "must mask this expression first");
2176 : 4 : auto vec = unary<int32_t, L / 2>(UNVOP_(ExtAddPairwiseS, I16x8ToI32x4));
2177 [ + - + - ]: 8 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
2178 : : vec.template extract_unsafe<0>(), // extract a single sum from vector to scalar
2179 : : vec);
2180 : 4 : }
2181 : 2 : PrimitiveExpr<uint32_t, L / 2> add_pairwise() requires unsigned_integral<T> and (sizeof(T) == 2) and (L > 1) {
2182 : : static_assert(L % 2 == 0, "must mask this expression first");
2183 : 2 : auto vec = unary<uint32_t, L / 2>(UNVOP_(ExtAddPairwiseU, I16x8ToI32x4));
2184 [ + - ]: 4 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
2185 : : vec.template extract_unsafe<0>(), // extract a single sum from vector to scalar
2186 : : vec);
2187 : 2 : }
2188 : :
2189 : : /*----- Bitwise operations ---------------------------------------------------------------------------------------*/
2190 : :
2191 [ # # # # : 4 : PrimitiveExpr operator~() requires integral<T> and (L == 1) { return PrimitiveExpr(T(-1)) xor *this; }
# # # # +
- - + # #
# # ]
2192 : 2 : PrimitiveExpr operator~() requires integral<T> and (L > 1) { return unary<T, L>(UNVOP_(Not, 128)); }
2193 : :
2194 : 4 : PrimitiveExpr clz() requires unsigned_integral<T> and (sizeof(T) >= 4) and (L == 1) {
2195 : 8 : return unary<T, L>(UNIOP_(Clz));
2196 : : }
2197 : 2 : PrimitiveExpr clz() requires unsigned_integral<T> and (sizeof(T) == 2) and (L == 1) {
2198 [ + - - + ]: 4 : return unary<T, L>(UNIOP_(Clz)) - PrimitiveExpr(16U); // the value is represented as I32
2199 : 0 : }
2200 : 2 : PrimitiveExpr clz() requires unsigned_integral<T> and (sizeof(T) == 1) and (L == 1) {
2201 [ + - - + ]: 4 : return unary<T, L>(UNIOP_(Clz)) - PrimitiveExpr(24U); // the value is represented as I32
2202 : 0 : }
2203 : 16 : PrimitiveExpr ctz() requires unsigned_integral<T> and (L == 1) { return unary<T, L>(UNIOP_(Ctz)); }
2204 : 16 : PrimitiveExpr popcnt() requires unsigned_integral<T> and (L == 1) { return unary<T, L>(UNIOP_(Popcnt)); }
2205 : 2 : PrimitiveExpr popcnt() requires unsigned_integral<T> and (sizeof(T) == 1) and (L > 1) {
2206 : 2 : return unary<T, L>(UNVOP_(Popcnt, I8x16));
2207 : : }
2208 : 2 : PrimitiveExpr popcnt() requires unsigned_integral<T> and (sizeof(T) == 2) and (L > 1) {
2209 : 2 : auto popcnt_on_I8x16 = this->unary<uint8_t, L * 2>(UNVOP_(Popcnt, I8x16));
2210 [ + - ]: 2 : return popcnt_on_I8x16.add_pairwise();
2211 : 2 : }
2212 : 2 : PrimitiveExpr popcnt() requires unsigned_integral<T> and (sizeof(T) == 4) and (L > 1) {
2213 : 2 : auto popcnt_on_I8x16 = this->unary<uint8_t, L * 4>(UNVOP_(Popcnt, I8x16));
2214 [ + - - + ]: 2 : return popcnt_on_I8x16.add_pairwise().add_pairwise();
2215 : 2 : }
2216 : :
2217 : : /** Concatenates the boolean values of `this` into a single mask. */
2218 : 0 : PrimitiveExpr<uint32_t, 1> bitmask() requires boolean<T> and (L > 1) {
2219 : 0 : auto bitmask = unary<uint32_t, 1>(UNVOP_(Bitmask, I8x16));
2220 [ # # ]: 0 : return M_CONSTEXPR_COND(L * sizeof(T) == 16, bitmask, bitmask bitand uint32_t((1U << L) - 1U)); // to remove unused lanes
2221 : 0 : }
2222 : : /** Concatenates the most significant bit of each value of `this` into a single mask. */
2223 : 10 : PrimitiveExpr<uint32_t, 1> bitmask() requires integral<T> and (L > 1) {
2224 : 20 : auto bitmask = unary<uint32_t, 1>(UNIVOP_(Bitmask));
2225 [ + - + - ]: 20 : return M_CONSTEXPR_COND(L * sizeof(T) == 16, bitmask, bitmask bitand uint32_t((1U << L) - 1U)); // to remove unused lanes
2226 : 10 : }
2227 : :
2228 : : /*----- Comparison operations ------------------------------------------------------------------------------------*/
2229 : :
2230 : 1032 : PrimitiveExpr<bool, L> eqz() requires integral<T> and (L == 1) { return unary<bool, L>(UNIOP_(EqZ)); }
2231 : :
2232 : : /*----- Logical operations ---------------------------------------------------------------------------------------*/
2233 : :
2234 : 8724 : PrimitiveExpr operator not() requires boolean<T> and (L == 1) { return unary<T, L>(UNIOP_(EqZ)); }
2235 : 44 : PrimitiveExpr operator not() requires boolean<T> and (L > 1) { return unary<T, L>(UNVOP_(Not, 128)); }
2236 : :
2237 : : /** Returns `true` iff any value is `true`. */
2238 : 28 : PrimitiveExpr<bool, 1> any_true() requires boolean<T> and (L > 1) {
2239 : : auto masked =
2240 [ # # # # : 56 : M_CONSTEXPR_COND(L * sizeof(T) == 16, *this, PrimitiveExpr(true) and *this); // to set the unused lanes to false
+ - - + ]
2241 [ # # + - ]: 28 : return masked.template unary<bool, 1>(UNVOP_(AnyTrue, 128));
2242 : 28 : }
2243 : : /** Returns `true` iff any value is non-zero. */
2244 : 8 : PrimitiveExpr<bool, 1> any_true() requires integral<T> and (L > 1) {
2245 : : auto masked =
2246 [ + - - + ]: 16 : M_CONSTEXPR_COND(L * sizeof(T) == 16, *this, PrimitiveExpr(T(-1)) bitand *this); // to set the unused lanes to 0
2247 [ + - ]: 8 : return masked.template unary<bool, 1>(UNVOP_(AnyTrue, 128));
2248 : 8 : }
2249 : : /** Returns `true` iff all values are `true`. */
2250 : 4 : PrimitiveExpr<bool, 1> all_true() requires boolean<T> and (L > 1) {
2251 : : std::array<uint8_t, 16> bytes;
2252 : 4 : auto it = std::fill_n(bytes.begin(), L, 0);
2253 : 4 : std::fill(it, bytes.end(), 0xff); // all bits to 1 represent true
2254 : : auto masked =
2255 [ + - - + : 8 : M_CONSTEXPR_COND(L * sizeof(T) == 16, *this, PrimitiveExpr(bytes) or *this); // to set the unused lanes to true
# # # # #
# # # ]
2256 [ + - # # : 4 : return masked.template unary<bool, 1>(UNVOP_(AllTrue, I8x16));
# # # # ]
2257 : 4 : }
2258 : : /** Returns `true` iff all values are non-zero. */
2259 : 8 : PrimitiveExpr<bool, 1> all_true() requires integral<T> and (L > 1) {
2260 : : std::array<uint8_t, 16> bytes;
2261 : 8 : auto it = std::fill_n(bytes.begin(), L, 0);
2262 : 8 : std::fill(it, bytes.end(), 1);
2263 : : auto masked =
2264 [ + - - + ]: 16 : M_CONSTEXPR_COND(L * sizeof(T) == 16, *this, PrimitiveExpr(bytes) bitor *this); // to set the unused lanes to 1
2265 [ + - + - ]: 16 : return masked.template unary<bool, 1>(UNIVOP_(AllTrue));
2266 : 8 : }
2267 : :
2268 : : /*----- Hashing operations ---------------------------------------------------------------------------------------*/
2269 : :
2270 : : PrimitiveExpr<uint64_t, L> hash() requires unsigned_integral<T> and (L == 1) { return *this; }
2271 [ # # # # : 0 : PrimitiveExpr<uint64_t, L> hash() requires signed_integral<T> and (L == 1) { return make_unsigned(); }
# # ]
2272 : 0 : PrimitiveExpr<uint64_t, L> hash() requires std::floating_point<T> and (sizeof(T) == 4) and (L == 1) {
2273 [ # # # # ]: 0 : return reinterpret<int32_t>().make_unsigned();
2274 : 0 : }
2275 : 0 : PrimitiveExpr<uint64_t, L> hash() requires std::floating_point<T> and (sizeof(T) == 8) and (L == 1) {
2276 [ # # ]: 0 : return reinterpret<int64_t>().make_unsigned();
2277 : 0 : }
2278 : 0 : PrimitiveExpr<uint64_t, L> hash() requires std::same_as<T, bool> and (L == 1) { return to<uint64_t>(); }
2279 : :
2280 : : #undef UNARY_VOP
2281 : : #undef UNFVOP_
2282 : : #undef UNIVOP_
2283 : : #undef UNVOP_
2284 : : #undef UNFOP_
2285 : : #undef UNIOP_
2286 : : #undef UNOP_
2287 : :
2288 : :
2289 : : /*------------------------------------------------------------------------------------------------------------------
2290 : : * Binary operations
2291 : : *----------------------------------------------------------------------------------------------------------------*/
2292 : :
2293 : : #define BINOP_(NAME, SIGN, TYPE) (::wasm::BinaryOp::NAME##SIGN##TYPE)
2294 : : #define BINIOP_(NAME, SIGN) [] { \
2295 : : if constexpr (sizeof(To) == 8) \
2296 : : return BINOP_(NAME,SIGN,Int64); \
2297 : : else if constexpr (sizeof(To) <= 4) \
2298 : : return BINOP_(NAME,SIGN,Int32); \
2299 : : else \
2300 : : M_unreachable("unsupported operation"); \
2301 : : } ()
2302 : : #define BINFOP_(NAME) [] { \
2303 : : if constexpr (sizeof(To) == 8) \
2304 : : return BINOP_(NAME,,Float64); \
2305 : : else if constexpr (sizeof(To) == 4) \
2306 : : return BINOP_(NAME,,Float32); \
2307 : : else \
2308 : : M_unreachable("unsupported operation"); \
2309 : : } ()
2310 : : #define BINARY_OP(NAME, SIGN) [] { \
2311 : : if constexpr (std::integral<To>) \
2312 : : return BINIOP_(NAME, SIGN); \
2313 : : else if constexpr (std::floating_point<To>) \
2314 : : return BINFOP_(NAME); \
2315 : : else \
2316 : : M_unreachable("unsupported operation"); \
2317 : : } ()
2318 : : #define BINVOP_(NAME, SIGN, TYPE) (::wasm::BinaryOp::NAME##SIGN##Vec##TYPE)
2319 : : #define BINIVOP_(NAME, SIGN) [] { \
2320 : : if constexpr (sizeof(To) == 8) \
2321 : : return BINVOP_(NAME,SIGN,I64x2); \
2322 : : else if constexpr (sizeof(To) == 4) \
2323 : : return BINVOP_(NAME,SIGN,I32x4); \
2324 : : else if constexpr (sizeof(To) == 2) \
2325 : : return BINVOP_(NAME,SIGN,I16x8); \
2326 : : else if constexpr (sizeof(To) == 1) \
2327 : : return BINVOP_(NAME,SIGN,I8x16); \
2328 : : else \
2329 : : M_unreachable("unsupported operation"); \
2330 : : } ()
2331 : : #define BINFVOP_(NAME) [] { \
2332 : : if constexpr (sizeof(To) == 8) \
2333 : : return BINVOP_(NAME,,F64x2); \
2334 : : else if constexpr (sizeof(To) == 4) \
2335 : : return BINVOP_(NAME,,F32x4); \
2336 : : else \
2337 : : M_unreachable("unsupported operation"); \
2338 : : } ()
2339 : : #define BINARY_VOP(NAME, SIGN) [] { \
2340 : : if constexpr (std::integral<To>) \
2341 : : return BINIVOP_(NAME, SIGN); \
2342 : : else if constexpr (std::floating_point<To>) \
2343 : : return BINFVOP_(NAME); \
2344 : : else \
2345 : : M_unreachable("unsupported operation"); \
2346 : : } ()
2347 : :
2348 : : /*----- Arithmetical operations ----------------------------------------------------------------------------------*/
2349 : :
2350 : : /** Adds `this` to \p other. */
2351 : : template<arithmetic U>
2352 : : requires arithmetically_combinable<T, U, L>
2353 : 1998 : auto operator+(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> {
2354 : : using To = common_type_t<T, U>;
2355 : : if constexpr (L * sizeof(To) <= 16)
2356 [ + - + - : 7992 : return binary<To, L, To, L>(M_CONSTEXPR_COND(L == 1, BINARY_OP(Add,), BINARY_VOP(Add,)), other);
+ - # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2357 : : else
2358 [ # # # # : 0 : return this->template to<To, L>().operator+(other.template to<To, L>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2359 : 0 : }
2360 : :
2361 : : /** Subtracts \p other from `this`. */
2362 : : template<arithmetic U>
2363 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2364 : 194 : auto operator-(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> {
2365 : : using To = common_type_t<T, U>;
2366 : : if constexpr (L * sizeof(To) <= 16)
2367 [ + - + - : 776 : return binary<To, L, To, L>(M_CONSTEXPR_COND(L == 1, BINARY_OP(Sub,), BINARY_VOP(Sub,)), other);
+ - + - #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2368 : : else
2369 [ # # # # : 0 : return this->template to<To, L>().operator-(other.template to<To, L>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2370 : 0 : }
2371 : :
2372 : : /** Multiplies `this` and \p other. */
2373 : : template<arithmetic U>
2374 : : requires arithmetically_combinable<T, U, L>
2375 : 294 : auto operator*(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires (L == 1) {
2376 : : using To = common_type_t<T, U>;
2377 [ + - # # : 882 : return binary<To, L, To, L>(BINARY_OP(Mul,), other);
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # +
- ]
2378 : 0 : }
2379 : : /** Multiplies `this` and \p other. */
2380 : : template<arithmetic U>
2381 : : requires arithmetically_combinable<T, U, L> and (sizeof(common_type_t<T, U>) != 1)
2382 : 4 : auto operator*(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires (L > 1) {
2383 : : using To = common_type_t<T, U>;
2384 : 8 : auto op = [](){
2385 : : if constexpr (std::integral<To>) {
2386 : : if constexpr (sizeof(To) == 8)
2387 : 0 : return BINVOP_(Mul,, I64x2);
2388 : : else if constexpr (sizeof(To) == 4)
2389 : 2 : return BINVOP_(Mul,, I32x4);
2390 : : else if constexpr (sizeof(To) == 2)
2391 : 0 : return BINVOP_(Mul,, I16x8);
2392 : : } else if (std::floating_point<To>) {
2393 : 4 : return BINFVOP_(Mul);
2394 : : }
2395 : : M_unreachable("unsupported operation");
2396 : : }();
2397 : : if constexpr (L * sizeof(To) <= 16)
2398 [ + - # # : 4 : return binary<To, L, To, L>(op, other);
# # # # #
# ]
2399 : : else
2400 [ # # # # : 0 : return this->template to<To, L>().operator*(other.template to<To, L>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2401 : 0 : }
2402 : : /** Multiplies `this` and \p other. */
2403 : : template<arithmetic U>
2404 : : requires arithmetically_combinable<T, U, L> and (sizeof(common_type_t<T, U>) == 1)
2405 : 0 : auto operator*(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires (L > 8) {
2406 : : static_assert(integral<U> and integral<T> and same_signedness<T, U>);
2407 : : static_assert(L == 16);
2408 : : using To = std::conditional_t<std::is_signed_v<T>, int16_t, uint16_t>;
2409 : 0 : auto op_low =
2410 : 0 : M_CONSTEXPR_COND(std::is_signed_v<T>, BINVOP_(ExtMulLow, S, I16x8), BINVOP_(ExtMulLow, U, I16x8));
2411 : 0 : auto op_high =
2412 : 0 : M_CONSTEXPR_COND(std::is_signed_v<T>, BINVOP_(ExtMulHigh, S, I16x8), BINVOP_(ExtMulHigh, U, I16x8));
2413 : 0 : auto this_cpy = this->clone();
2414 [ # # ]: 0 : auto other_cpy = other.clone();
2415 : 0 : auto referenced_bits_low = this_cpy.referenced_bits(); // moved
2416 : 0 : auto referenced_bits_high = this->referenced_bits(); // moved
2417 : 0 : referenced_bits_low.splice(referenced_bits_low.end(), other_cpy.referenced_bits());
2418 : 0 : referenced_bits_high.splice(referenced_bits_high.end(), other.referenced_bits());
2419 [ # # ]: 0 : PrimitiveExpr<To, 8> low(
2420 [ # # # # : 0 : /* expr= */ Module::Builder().makeBinary(op_low, this_cpy.expr(), other_cpy.expr()),
# # # # ]
2421 : 0 : /* referenced_bits= */ std::move(referenced_bits_low)
2422 : : );
2423 [ # # ]: 0 : PrimitiveExpr<To, 8> high(
2424 [ # # # # : 0 : /* expr= */ Module::Builder().makeBinary(op_high, this->expr(), other.expr()),
# # # # ]
2425 : 0 : /* referenced_bits= */ std::move(referenced_bits_high)
2426 : : );
2427 : : /* Shuffle to select the lower byte of each lane since integer narrowing would be performed saturating, i.e.
2428 : : * instead of overflows the values are mapped to the extremes of their domain. */
2429 : 0 : auto indices = std::to_array<uint8_t>({ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 });
2430 [ # # # # : 0 : return PrimitiveExpr<common_type_t<T, U>, L>(ShuffleBytes(low, high, indices).move());
# # ]
2431 : 0 : }
2432 : :
2433 : : /** Divides `this` by \p other. */
2434 : : template<arithmetic U>
2435 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2436 : 138 : auto operator/(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires (L == 1) {
2437 : : using To = common_type_t<T, U>;
2438 [ + - + - : 690 : return binary<To, L, To, L>(M_CONSTEXPR_COND(std::is_signed_v<To>, BINARY_OP(Div, S), BINARY_OP(Div, U)),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
# # # # #
# # # # #
# # # # #
# + - + -
# # # # #
# # # # #
# # # # #
# + - +
- ]
2439 : 138 : other);
2440 : 0 : }
2441 : : /** Divides `this` by \p other. */
2442 : : template<std::floating_point U>
2443 : : requires arithmetically_combinable<T, U, L>
2444 : 2 : auto operator/(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L>
2445 : : requires std::floating_point<T> and (L > 1) {
2446 : : using To = common_type_t<T, U>;
2447 : : if constexpr (L * sizeof(To) <= 16)
2448 [ + - # # ]: 4 : return binary<To, L, To, L>(BINFVOP_(Div), other);
2449 : : else
2450 [ # # # # ]: 0 : return this->template to<To, L>().operator/(other.template to<To, L>());
2451 : 0 : }
2452 : :
2453 : : /** Computes the remainder of dividing `this` by \p other. */
2454 : : template<integral U>
2455 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2456 : 460 : auto operator%(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L>
2457 : : requires integral<T> and (L == 1) {
2458 : : using To = common_type_t<T, U>;
2459 [ + - # # : 1380 : return binary<To, L, To, L>(M_CONSTEXPR_COND(std::is_signed_v<To>, BINIOP_(Rem, S), BINIOP_(Rem, U)), other);
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # #
# ]
2460 : 0 : }
2461 : :
2462 : : /** Copy the sign bit of \p other to `this`. */
2463 : : template<std::floating_point U>
2464 : : requires arithmetically_combinable<T, U, L>
2465 : 8 : auto copy_sign(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L>
2466 : : requires std::floating_point<T> and (L == 1) {
2467 : : using To = common_type_t<T, U>;
2468 [ + - ]: 16 : return binary<To, L, To, L>(BINFOP_(CopySign), other);
2469 : 0 : }
2470 : :
2471 : : /** Computes the minimum of `this` and \p other. */
2472 : : template<std::floating_point U>
2473 : : requires arithmetically_combinable<T, U, L>
2474 : 6 : auto min(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires std::floating_point<T> {
2475 : : using To = common_type_t<T, U>;
2476 : : if constexpr (L * sizeof(To) <= 16)
2477 [ # # + - : 18 : return binary<To, L, To, L>(M_CONSTEXPR_COND(L == 1, BINFOP_(Min), BINFVOP_(Min)), other); // XXX: or PMin?
+ - # # ]
2478 : : else
2479 : : return this->template to<To, L>().min(other.template to<To, L>());
2480 : 0 : }
2481 : : /** Computes the minimum of `this` and \p other. */
2482 : : template<integral U>
2483 : : requires arithmetically_combinable<T, U, L> and (sizeof(common_type_t<T, U>) != 8)
2484 : 2 : auto min(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires integral<T> and (L > 1) {
2485 : : using To = common_type_t<T, U>;
2486 : 4 : auto op = [](){
2487 : : if constexpr (sizeof(To) == 4)
2488 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Min, S, I32x4), BINVOP_(Min, U, I32x4));
2489 : : else if constexpr (sizeof(To) == 2)
2490 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Min, S, I16x8), BINVOP_(Min, U, I16x8));
2491 : : else if constexpr (sizeof(To) == 1)
2492 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Min, S, I8x16), BINVOP_(Min, U, I8x16));
2493 : : M_unreachable("unsupported operation");
2494 : : }();
2495 : : if constexpr (L * sizeof(To) <= 16)
2496 [ + - # # : 2 : return binary<To, L, To, L>(op, other);
# # ]
2497 : : else
2498 : : return this->template to<To, L>().min(other.template to<To, L>());
2499 : 0 : }
2500 : :
2501 : : /** Computes the maximum of `this` and \p other. */
2502 : : template<std::floating_point U>
2503 : : requires arithmetically_combinable<T, U, L>
2504 : 6 : auto max(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires std::floating_point<T> {
2505 : : using To = common_type_t<T, U>;
2506 : : if constexpr (L * sizeof(To) <= 16)
2507 [ # # + - : 18 : return binary<To, L, To, L>(M_CONSTEXPR_COND(L == 1, BINFOP_(Max), BINFVOP_(Max)), other); // XXX: or PMax?
+ - # # ]
2508 : : else
2509 : : return this->template to<To, L>().max(other.template to<To, L>());
2510 : 0 : }
2511 : : /** Computes the maximum of `this` and \p other. */
2512 : : template<integral U>
2513 : : requires arithmetically_combinable<T, U, L> and (sizeof(common_type_t<T, U>) != 8)
2514 : 2 : auto max(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires integral<T> and (L > 1) {
2515 : : using To = common_type_t<T, U>;
2516 : 4 : auto op = [](){
2517 : : if constexpr (sizeof(To) == 4)
2518 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Max, S, I32x4), BINVOP_(Max, U, I32x4));
2519 : : else if constexpr (sizeof(To) == 2)
2520 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Max, S, I16x8), BINVOP_(Max, U, I16x8));
2521 : : else if constexpr (sizeof(To) == 1)
2522 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Max, S, I8x16), BINVOP_(Max, U, I8x16));
2523 : : M_unreachable("unsupported operation");
2524 : : }();
2525 : : if constexpr (L * sizeof(To) <= 16)
2526 [ + - # # : 2 : return binary<To, L, To, L>(op, other);
# # ]
2527 : : else
2528 : : return this->template to<To, L>().max(other.template to<To, L>());
2529 : 0 : }
2530 : :
2531 : : /** Computes the (ceiled) average of `this` and \p other. */
2532 : : template<unsigned_integral U>
2533 : : requires arithmetically_combinable<T, U, L> and (sizeof(common_type_t<T, U>) <= 2)
2534 : 2 : auto avg(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L>
2535 : : requires unsigned_integral<T> and (L > 1) {
2536 : : using To = common_type_t<T, U>;
2537 : 4 : auto op = [](){
2538 : : if constexpr (sizeof(To) == 2)
2539 : 2 : return BINVOP_(Avgr, U, I16x8);
2540 : : else if constexpr (sizeof(To) == 1)
2541 : : return BINVOP_(Avgr, U, I8x16);
2542 : : M_unreachable("unsupported operation");
2543 : : }();
2544 : : if constexpr (L * sizeof(To) <= 16)
2545 [ + - ]: 2 : return binary<To, L, To, L>(op, other);
2546 : : else
2547 : : return this->template to<To, L>().avg(other.template to<To, L>());
2548 : 0 : }
2549 : :
2550 : : /*----- Bitwise operations ---------------------------------------------------------------------------------------*/
2551 : :
2552 : : /** Computes the bitwise *and* of `this` and \p other. */
2553 : : template<std::integral U>
2554 : : requires arithmetically_combinable<T, U, L>
2555 : 1676 : auto operator bitand(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires std::integral<T> {
2556 : : using To = common_type_t<T, U>;
2557 : : if constexpr (L * sizeof(To) <= 16)
2558 [ + - + - : 4954 : return binary<To, L, To, L>(M_CONSTEXPR_COND(L == 1, BINIOP_(And,), BINVOP_(And,, 128)), other);
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# + - ]
2559 : : else
2560 : : return this->template to<To, L>().operator bitand(other.template to<To, L>());
2561 : 0 : }
2562 : :
2563 : : /** Computes the bitwise *or* of `this` and \p other. */
2564 : : template<std::integral U>
2565 : : requires arithmetically_combinable<T, U, L>
2566 : 684 : auto operator bitor(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires std::integral<T> {
2567 : : using To = common_type_t<T, U>;
2568 : : if constexpr (L * sizeof(To) <= 16)
2569 [ + - + - : 2010 : return binary<To, L, To, L>(M_CONSTEXPR_COND(L == 1, BINIOP_(Or,), BINVOP_(Or,, 128)), other);
+ - + - +
- # # # #
# # # # ]
2570 : : else
2571 : : return this->template to<To, L>().operator bitor(other.template to<To, L>());
2572 : 0 : }
2573 : :
2574 : : /** Computes the (bitwise) *xor* of `this` and \p other. */
2575 : : template<std::integral U>
2576 : : requires arithmetically_combinable<T, U, L>
2577 : 44 : auto operator xor(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires std::integral<T> {
2578 : : using To = common_type_t<T, U>;
2579 : : if constexpr (L * sizeof(To) <= 16)
2580 [ + - + - : 114 : return binary<To, L, To, L>(M_CONSTEXPR_COND(L == 1, BINIOP_(Xor,), BINVOP_(Xor,, 128)), other);
+ - # # #
# # # # #
# # + - #
# # # # #
# # # # #
# ]
2581 : : else
2582 : : return this->template to<To, L>().operator xor(other.template to<To, L>());
2583 : 0 : }
2584 : :
2585 : : /** Shifts `this` *left* by \p other. */
2586 : : template<integral U>
2587 : : requires arithmetically_combinable<T, U, L>
2588 : 1334 : auto operator<<(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L>
2589 : : requires integral<T> and (L == 1) {
2590 : : using To = common_type_t<T, U>;
2591 : : if constexpr (sizeof(To) >= 4)
2592 [ + - + - : 2660 : return binary<To, L, To, L>(BINIOP_(Shl,), other);
+ - + - ]
2593 : : else if constexpr (sizeof(To) == 2)
2594 : : return binary<To, L, To, L>(BINOP_(Shl,, Int32), other) bitand PrimitiveExpr<To, 1>(0xffff);
2595 : : else if constexpr (sizeof(To) == 1)
2596 [ + - + - : 4 : return binary<To, L, To, L>(BINOP_(Shl,, Int32), other) bitand PrimitiveExpr<To, 1>(0xff);
- + ]
2597 : : else
2598 : : M_unreachable("unsupported operation");
2599 : 0 : }
2600 : : /** Shifts `this` *left* by \p other. */
2601 : : template<integral U>
2602 : : requires requires (PrimitiveExpr<U, 1> e) {
2603 : : PrimitiveExpr<std::conditional_t<std::is_signed_v<T>, int32_t, uint32_t>, 1>(e);
2604 : : }
2605 : 34 : PrimitiveExpr operator<<(PrimitiveExpr<U, 1> other)
2606 : : requires integral<T> and (L > 1) {
2607 : : using Op = std::conditional_t<std::is_signed_v<T>, int32_t, uint32_t>;
2608 : 68 : auto op = [](){
2609 : : if constexpr (sizeof(T) == 8)
2610 : : return ::wasm::SIMDShiftOp::ShlVecI64x2;
2611 : : else if constexpr (sizeof(T) == 4)
2612 : : return ::wasm::SIMDShiftOp::ShlVecI32x4;
2613 : : else if constexpr (sizeof(T) == 2)
2614 : : return ::wasm::SIMDShiftOp::ShlVecI16x8;
2615 : : else if constexpr (sizeof(T) == 1)
2616 : 34 : return ::wasm::SIMDShiftOp::ShlVecI8x16;
2617 : : else
2618 : : M_unreachable("unsupported operation");
2619 : : }();
2620 : 34 : auto referenced_bits = this->referenced_bits(); // moved
2621 [ + - ]: 34 : referenced_bits.splice(referenced_bits.end(), other.referenced_bits());
2622 [ - + # # ]: 34 : return PrimitiveExpr(
2623 [ + - + - : 34 : /* expr= */ Module::Builder().makeSIMDShift(op, this->expr(), PrimitiveExpr<Op, 1>(other).expr()),
+ - + - +
- # # # #
# # # # #
# ]
2624 : 34 : /* referenced_bits= */ std::move(referenced_bits)
2625 : : );
2626 : 34 : }
2627 : :
2628 : : /** Shifts `this` *right* by \p other. */
2629 : : template<integral U>
2630 : : requires arithmetically_combinable<T, U, L>
2631 : 96 : auto operator>>(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L>
2632 : : requires integral<T> and (L == 1) {
2633 : : using To = common_type_t<T, U>;
2634 [ + - + - : 288 : return binary<To, L, To, L>(M_CONSTEXPR_COND(std::is_signed_v<T>, BINIOP_(Shr, S), BINIOP_(Shr, U)), other);
+ - ]
2635 : 0 : }
2636 : : /** Shifts `this` *right* by \p other. */
2637 : : template<integral U>
2638 : : requires requires (PrimitiveExpr<U, 1> e) {
2639 : : PrimitiveExpr<std::conditional_t<std::is_signed_v<T>, int32_t, uint32_t>, 1>(e);
2640 : : }
2641 : 2 : PrimitiveExpr operator>>(PrimitiveExpr<U, 1> other)
2642 : : requires integral<T> and (L > 1) {
2643 : : using Op = std::conditional_t<std::is_signed_v<T>, int32_t, uint32_t>;
2644 : 4 : auto op = [](){
2645 : : if constexpr (sizeof(T) == 8)
2646 : : return M_CONSTEXPR_COND(std::is_signed_v<T>, ::wasm::SIMDShiftOp::ShrSVecI64x2,
2647 : : ::wasm::SIMDShiftOp::ShrUVecI64x2);
2648 : : else if constexpr (sizeof(T) == 4)
2649 : : return M_CONSTEXPR_COND(std::is_signed_v<T>, ::wasm::SIMDShiftOp::ShrSVecI32x4,
2650 : : ::wasm::SIMDShiftOp::ShrUVecI32x4);
2651 : : else if constexpr (sizeof(T) == 2)
2652 : : return M_CONSTEXPR_COND(std::is_signed_v<T>, ::wasm::SIMDShiftOp::ShrSVecI16x8,
2653 : : ::wasm::SIMDShiftOp::ShrUVecI16x8);
2654 : : else if constexpr (sizeof(T) == 1)
2655 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<T>, ::wasm::SIMDShiftOp::ShrSVecI8x16,
2656 : : ::wasm::SIMDShiftOp::ShrUVecI8x16);
2657 : : else
2658 : : M_unreachable("unsupported operation");
2659 : : }();
2660 : 2 : auto referenced_bits = this->referenced_bits(); // moved
2661 : 2 : referenced_bits.splice(referenced_bits.end(), other.referenced_bits());
2662 [ - + ]: 2 : return PrimitiveExpr(
2663 [ + - + - : 2 : /* expr= */ Module::Builder().makeSIMDShift(op, this->expr(), PrimitiveExpr<Op, 1>(other).expr()),
+ - + - +
- ]
2664 : 2 : /* referenced_bits= */ std::move(referenced_bits)
2665 : : );
2666 : 2 : }
2667 : :
2668 : : /** Rotates `this` *left* by \p other. */
2669 : : template<integral U>
2670 : : requires arithmetically_combinable<T, U, L> and (sizeof(common_type_t<T, U>) >= 4)
2671 : 4 : auto rotl(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires integral<T> and (L == 1) {
2672 : : using To = common_type_t<T, U>;
2673 [ + - ]: 8 : return binary<To, L, To, L>(BINIOP_(RotL,), other);
2674 : 0 : }
2675 : :
2676 : : /** Rotates `this` *right* by \p other. */
2677 : : template<integral U>
2678 : : requires arithmetically_combinable<T, U, L> and (sizeof(common_type_t<T, U>) >= 4)
2679 : 4 : auto rotr(PrimitiveExpr<U, L> other) -> PrimitiveExpr<common_type_t<T, U>, L> requires integral<T> and (L == 1) {
2680 : : using To = common_type_t<T, U>;
2681 [ + - ]: 8 : return binary<To, L, To, L>(BINIOP_(RotR,), other);
2682 : 0 : }
2683 : :
2684 : : /*----- Comparison operations ------------------------------------------------------------------------------------*/
2685 : :
2686 : : /** Checks whether `this` equals \p other. */
2687 : : template<dsl_primitive U>
2688 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2689 : 1980 : PrimitiveExpr<bool, L> operator==(PrimitiveExpr<U, L> other) {
2690 : : using To = common_type_t<T, U>;
2691 : 1980 : constexpr std::size_t ToL = L == 1 ? L : L * sizeof(T);
2692 : : if constexpr (L * sizeof(To) <= 16) {
2693 [ + - + - : 7920 : auto cmp = binary<bool, ToL, To, L>(M_CONSTEXPR_COND(L == 1, BINARY_OP(Eq,), BINARY_VOP(Eq,)), other);
+ - + - +
- + - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - +
- ]
2694 : : std::array<uint8_t, L> indices;
2695 [ + + + + : 3964 : for (std::size_t idx = 0; idx < L; ++idx)
+ + + + +
+ + + + +
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ + + +
+ ]
2696 : 1984 : indices[idx] = idx * sizeof(To);
2697 [ - + - + : 3960 : return M_CONSTEXPR_COND(L == 1 or sizeof(To) == 1, cmp, cmp.swizzle_bytes(indices));
- + - + -
+ - + - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ - + -
+ ]
2698 : 1980 : } else {
2699 [ # # # # : 0 : return this->template to<To, L>().operator==(other.template to<To, L>());
# # # # #
# # # ]
2700 : : }
2701 : 1980 : }
2702 : :
2703 : : /** Checks whether `this` unequal to \p other. */
2704 : : template<dsl_primitive U>
2705 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2706 : 1184 : PrimitiveExpr<bool, L> operator!=(PrimitiveExpr<U, L> other) {
2707 : : using To = common_type_t<T, U>;
2708 : 1184 : constexpr std::size_t ToL = L == 1 ? L : L * sizeof(T);
2709 : : if constexpr (L * sizeof(To) <= 16) {
2710 [ + - + - : 4736 : auto cmp = binary<bool, ToL, To, L>(M_CONSTEXPR_COND(L == 1, BINARY_OP(Ne,), BINARY_VOP(Ne,)), other);
+ - # # +
- + - # #
# # # # +
- + - # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2711 : : std::array<uint8_t, L> indices;
2712 [ + + + + : 2858 : for (std::size_t idx = 0; idx < L; ++idx)
+ + # # +
+ + + # #
# # # # +
+ + + # #
+ + + + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2713 : 1674 : indices[idx] = idx * sizeof(To);
2714 [ - + - + : 2368 : return M_CONSTEXPR_COND(L == 1 or sizeof(To) == 1, cmp, cmp.swizzle_bytes(indices));
- + # # -
+ - + # #
# # # # -
+ - + # #
- + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2715 : 1184 : } else {
2716 [ # # # # : 0 : return this->template to<To, L>().operator!=(other.template to<To, L>());
# # # # #
# # # ]
2717 : : }
2718 : 1184 : }
2719 : :
2720 : : /** Checks whether `this` less than \p other. */
2721 : : template<arithmetic U>
2722 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2723 : 258 : PrimitiveExpr<bool, L> operator<(PrimitiveExpr<U, L> other) requires arithmetic<T> {
2724 : : using To = common_type_t<T, U>;
2725 : 258 : constexpr std::size_t ToL = L == 1 ? L : L * sizeof(T);
2726 : 516 : auto op = [](){
2727 : : if constexpr (L == 1) {
2728 : 1008 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINARY_OP(Lt, S), BINARY_OP(Lt, U));
2729 : : } else {
2730 : : if constexpr (std::integral<To>) {
2731 : : if constexpr (sizeof(To) == 8)
2732 : 2 : return BINVOP_(Lt, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
2733 : : else if constexpr (sizeof(To) == 4)
2734 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Lt, S, I32x4), BINVOP_(Lt, U, I32x4));
2735 : : else if constexpr (sizeof(To) == 2)
2736 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Lt, S, I16x8), BINVOP_(Lt, U, I16x8));
2737 : : else if constexpr (sizeof(To) == 1)
2738 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Lt, S, I8x16), BINVOP_(Lt, U, I8x16));
2739 : : } else if (std::floating_point<To>) {
2740 : 4 : return BINFVOP_(Lt);
2741 : : }
2742 : : }
2743 : : M_unreachable("unsupported operation");
2744 : : }();
2745 : : if constexpr (L * sizeof(To) <= 16) {
2746 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
2747 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
2748 : 258 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
2749 : : auto _this =
2750 [ + - ]: 516 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
2751 : : auto _other =
2752 [ + - + - : 516 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
+ - + - +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- # # # #
+ - ]
2753 [ + - + - : 258 : auto cmp = _this.template binary<bool, ToL, To, L>(op, _other);
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- # # # #
# # # # +
- + - ]
2754 : : std::array<uint8_t, L> indices;
2755 [ + + + + : 530 : for (std::size_t idx = 0; idx < L; ++idx)
+ + + + +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ # # # #
+ + ]
2756 : 272 : indices[idx] = idx * sizeof(To);
2757 [ - + - + : 516 : return M_CONSTEXPR_COND(L == 1 or sizeof(To) == 1, cmp, cmp.swizzle_bytes(indices));
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # # #
- + ]
2758 : 258 : } else {
2759 [ # # # # : 0 : return this->template to<To, L>().operator<(other.template to<To, L>());
# # # # #
# # # ]
2760 : : }
2761 : 258 : }
2762 : :
2763 : : /** Checks whether `this` less than or equals to \p other. */
2764 : : template<arithmetic U>
2765 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2766 : 36 : PrimitiveExpr<bool, L> operator<=(PrimitiveExpr<U, L> other) requires arithmetic<T> {
2767 : : using To = common_type_t<T, U>;
2768 : 36 : constexpr std::size_t ToL = L == 1 ? L : L * sizeof(T);
2769 : 72 : auto op = [](){
2770 : : if constexpr (L == 1) {
2771 : 120 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINARY_OP(Le, S), BINARY_OP(Le, U));
2772 : : } else {
2773 : : if constexpr (std::integral<To>) {
2774 : : if constexpr (sizeof(To) == 8)
2775 : 2 : return BINVOP_(Le, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
2776 : : else if constexpr (sizeof(To) == 4)
2777 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Le, S, I32x4), BINVOP_(Le, U, I32x4));
2778 : : else if constexpr (sizeof(To) == 2)
2779 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Le, S, I16x8), BINVOP_(Le, U, I16x8));
2780 : : else if constexpr (sizeof(To) == 1)
2781 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Le, S, I8x16), BINVOP_(Le, U, I8x16));
2782 : : } else if (std::floating_point<To>) {
2783 : 4 : return BINFVOP_(Le);
2784 : : }
2785 : : }
2786 : : M_unreachable("unsupported operation");
2787 : : }();
2788 : : if constexpr (L * sizeof(To) <= 16) {
2789 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
2790 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
2791 : 36 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
2792 : : auto _this =
2793 [ + - ]: 72 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
2794 : : auto _other =
2795 [ + - + - : 72 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
+ - + - +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- # # ]
2796 [ + - + - : 36 : auto cmp = _this.template binary<bool, ToL, To, L>(op, _other);
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- # # #
# ]
2797 : : std::array<uint8_t, L> indices;
2798 [ + + + + : 86 : for (std::size_t idx = 0; idx < L; ++idx)
+ + + + +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ # # ]
2799 : 50 : indices[idx] = idx * sizeof(To);
2800 [ - + - + : 72 : return M_CONSTEXPR_COND(L == 1 or sizeof(To) == 1, cmp, cmp.swizzle_bytes(indices));
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # ]
2801 : 36 : } else {
2802 [ # # # # : 0 : return this->template to<To, L>().operator<=(other.template to<To, L>());
# # # # #
# # # ]
2803 : : }
2804 : 36 : }
2805 : :
2806 : : /** Checks whether `this` greater than to \p other. */
2807 : : template<arithmetic U>
2808 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2809 : 338 : PrimitiveExpr<bool, L> operator>(PrimitiveExpr<U, L> other) requires arithmetic<T> {
2810 : : using To = common_type_t<T, U>;
2811 : 338 : constexpr std::size_t ToL = L == 1 ? L : L * sizeof(T);
2812 : 676 : auto op = [](){
2813 : : if constexpr (L == 1) {
2814 : 1328 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINARY_OP(Gt, S), BINARY_OP(Gt, U));
2815 : : } else {
2816 : : if constexpr (std::integral<To>) {
2817 : : if constexpr (sizeof(To) == 8)
2818 : 2 : return BINVOP_(Gt, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
2819 : : else if constexpr (sizeof(To) == 4)
2820 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Gt, S, I32x4), BINVOP_(Gt, U, I32x4));
2821 : : else if constexpr (sizeof(To) == 2)
2822 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Gt, S, I16x8), BINVOP_(Gt, U, I16x8));
2823 : : else if constexpr (sizeof(To) == 1)
2824 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Gt, S, I8x16), BINVOP_(Gt, U, I8x16));
2825 : : } else if (std::floating_point<To>) {
2826 : 4 : return BINFVOP_(Gt);
2827 : : }
2828 : : }
2829 : : M_unreachable("unsupported operation");
2830 : : }();
2831 : : if constexpr (L * sizeof(To) <= 16) {
2832 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
2833 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
2834 : 338 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
2835 : : auto _this =
2836 [ + - ]: 676 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
2837 : : auto _other =
2838 [ + - + - : 676 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
+ - + - +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- # # +
- ]
2839 [ + - + - : 338 : auto cmp = _this.template binary<bool, ToL, To, L>(op, _other);
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- # # # #
+ - + - ]
2840 : : std::array<uint8_t, L> indices;
2841 [ + + + + : 690 : for (std::size_t idx = 0; idx < L; ++idx)
+ + + + +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ # # +
+ ]
2842 : 352 : indices[idx] = idx * sizeof(To);
2843 [ - + - + : 676 : return M_CONSTEXPR_COND(L == 1 or sizeof(To) == 1, cmp, cmp.swizzle_bytes(indices));
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # -
+ ]
2844 : 338 : } else {
2845 [ # # # # : 0 : return this->template to<To, L>().operator>(other.template to<To, L>());
# # # # #
# # # ]
2846 : : }
2847 : 338 : }
2848 : :
2849 : : /** Checks whether `this` greater than or equals to \p other. */
2850 : : template<arithmetic U>
2851 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
2852 : 232 : PrimitiveExpr<bool, L> operator>=(PrimitiveExpr<U, L> other) requires arithmetic<T> {
2853 : : using To = common_type_t<T, U>;
2854 : 232 : constexpr std::size_t ToL = L == 1 ? L : L * sizeof(T);
2855 : 464 : auto op = [](){
2856 : : if constexpr (L == 1) {
2857 : 904 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINARY_OP(Ge, S), BINARY_OP(Ge, U));
2858 : : } else {
2859 : : if constexpr (std::integral<To>) {
2860 : : if constexpr (sizeof(To) == 8)
2861 : 2 : return BINVOP_(Ge, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
2862 : : else if constexpr (sizeof(To) == 4)
2863 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Ge, S, I32x4), BINVOP_(Ge, U, I32x4));
2864 : : else if constexpr (sizeof(To) == 2)
2865 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Ge, S, I16x8), BINVOP_(Ge, U, I16x8));
2866 : : else if constexpr (sizeof(To) == 1)
2867 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Ge, S, I8x16), BINVOP_(Ge, U, I8x16));
2868 : : } else if (std::floating_point<To>) {
2869 : 4 : return BINFVOP_(Ge);
2870 : : }
2871 : : }
2872 : : M_unreachable("unsupported operation");
2873 : : }();
2874 : : if constexpr (L * sizeof(To) <= 16) {
2875 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
2876 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
2877 : 232 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
2878 : : auto _this =
2879 [ + - ]: 464 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
2880 : : auto _other =
2881 [ + - + - : 464 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
+ - + - +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- # # ]
2882 [ + - + - : 232 : auto cmp = _this.template binary<bool, ToL, To, L>(op, _other);
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- # # #
# ]
2883 : : std::array<uint8_t, L> indices;
2884 [ + + + + : 478 : for (std::size_t idx = 0; idx < L; ++idx)
+ + + + +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ # # ]
2885 : 246 : indices[idx] = idx * sizeof(To);
2886 [ - + - + : 464 : return M_CONSTEXPR_COND(L == 1 or sizeof(To) == 1, cmp, cmp.swizzle_bytes(indices));
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # ]
2887 : 232 : } else {
2888 [ # # # # : 0 : return this->template to<To, L>().operator>=(other.template to<To, L>());
# # # # #
# # # ]
2889 : : }
2890 : 232 : }
2891 : :
2892 : : /*----- Logical operations ---------------------------------------------------------------------------------------*/
2893 : :
2894 : : /** Computes the logical conjunction (`and`) of `this` and \p other. */
2895 : : template<boolean U>
2896 : 626 : PrimitiveExpr<bool, L> operator and(PrimitiveExpr<U, L> other) requires boolean<T> {
2897 [ + - # # : 1252 : return binary<T, L, T, L>(M_CONSTEXPR_COND(L == 1, BINOP_(And,,Int32), BINVOP_(And,, 128)), other);
+ - + - #
# ]
2898 : 0 : }
2899 : :
2900 : : /** Computes the logical conjunction (`and`) of `this` and the logical negation (`not`) of \p other. */
2901 : : template<boolean U>
2902 : 52 : PrimitiveExpr<bool, L> and_not(PrimitiveExpr<U, L> other) requires boolean<T> and (L > 1) {
2903 [ + - + - ]: 52 : return binary<T, L, T, L>(BINVOP_(AndNot,, 128), other);
2904 : 0 : }
2905 : :
2906 : : /** Computes the logical disjunction (`or`) of `this` and \p other. */
2907 : : template<boolean U>
2908 : 418 : PrimitiveExpr<bool, L> operator or(PrimitiveExpr<U, L> other) requires boolean<T> {
2909 [ + - + - : 836 : return binary<T, L, T, L>(M_CONSTEXPR_COND(L == 1, BINOP_(Or,,Int32), BINVOP_(Or,, 128)), other);
# # # # #
# ]
2910 : 0 : }
2911 : :
2912 : : #undef BINARY_VOP
2913 : : #undef BINFVOP_
2914 : : #undef BINIVOP_
2915 : : #undef BINVOP_
2916 : : #undef BINARY_OP
2917 : : #undef BINFOP_
2918 : : #undef BINIOP_
2919 : : #undef BINOP_
2920 : :
2921 : :
2922 : : /*------------------------------------------------------------------------------------------------------------------
2923 : : * Modifications
2924 : : *----------------------------------------------------------------------------------------------------------------*/
2925 : :
2926 : : private:
2927 : : /** Extracts the \tparam M -th value of the underlying 128 bit vector of `this`. Special care must be taken as
2928 : : * this method *must* not be called on scalar expressions and the extracted value might not be one of the first
2929 : : * (defined) `L` values. */
2930 : : template<std::size_t M>
2931 : : requires (M * sizeof(T) < 16)
2932 : 24 : PrimitiveExpr<T, 1> extract_unsafe() {
2933 : 48 : auto op = [](){
2934 : : if constexpr (std::integral<T>) {
2935 : : if constexpr (sizeof(T) == 8)
2936 : 0 : return ::wasm::SIMDExtractOp::ExtractLaneVecI64x2;
2937 : : else if constexpr (sizeof(T) == 4)
2938 : 8 : return ::wasm::SIMDExtractOp::ExtractLaneVecI32x4;
2939 : : else if constexpr (sizeof(T) == 2)
2940 : 12 : return M_CONSTEXPR_COND(std::is_signed_v<T>, ::wasm::SIMDExtractOp::ExtractLaneSVecI16x8,
2941 : : ::wasm::SIMDExtractOp::ExtractLaneUVecI16x8);
2942 : : else if constexpr (sizeof(T) == 1)
2943 : 20 : return M_CONSTEXPR_COND(std::is_signed_v<T>, ::wasm::SIMDExtractOp::ExtractLaneSVecI8x16,
2944 : : ::wasm::SIMDExtractOp::ExtractLaneUVecI8x16);
2945 : : } else if (std::floating_point<T>) {
2946 : : if constexpr (sizeof(T) == 8)
2947 : 0 : return ::wasm::SIMDExtractOp::ExtractLaneVecF64x2;
2948 : : else if constexpr (sizeof(T) == 4)
2949 : 0 : return ::wasm::SIMDExtractOp::ExtractLaneVecF32x4;
2950 : : }
2951 : : M_unreachable("unsupported operation");
2952 : : }();
2953 [ + - # # : 24 : auto extracted = PrimitiveExpr<T, 1>(
+ - # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
2954 : 24 : /* expr= */ Module::Builder().makeSIMDExtract(op, expr(), M),
2955 : 24 : /* referenced_bits= */ referenced_bits() // moved
2956 : : );
2957 [ - + # # : 48 : return M_CONSTEXPR_COND(boolean<T>, extracted != false, extracted);
- + # # #
# - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
2958 : 24 : }
2959 : : public:
2960 : : /** Extracts the \tparam M -th value of `this`. */
2961 : : template<std::size_t M>
2962 : : requires (M < L)
2963 : 16 : PrimitiveExpr<T, 1> extract() requires (L > 1) { return extract_unsafe<M>(); }
2964 : :
2965 : : /** Replaces the \tparam M -th value of `this` with \p value. */
2966 : : template<std::size_t M, primitive_convertible U>
2967 : : requires (M < L) and
2968 : : requires (primitive_expr_t<U> u) { PrimitiveExpr<T, 1>(u); }
2969 : 2 : PrimitiveExpr replace(U &&_value) requires (L > 1) {
2970 : 4 : auto op = [](){
2971 : : if constexpr (std::integral<T>) {
2972 : : if constexpr (sizeof(T) == 8)
2973 : : return ::wasm::SIMDReplaceOp::ReplaceLaneVecI64x2;
2974 : : else if constexpr (sizeof(T) == 4)
2975 : 2 : return ::wasm::SIMDReplaceOp::ReplaceLaneVecI32x4;
2976 : : else if constexpr (sizeof(T) == 2)
2977 : : return ::wasm::SIMDReplaceOp::ReplaceLaneVecI16x8;
2978 : : else if constexpr (sizeof(T) == 1)
2979 : 0 : return ::wasm::SIMDReplaceOp::ReplaceLaneVecI8x16;
2980 : : } else if (std::floating_point<T>) {
2981 : : if constexpr (sizeof(T) == 8)
2982 : : return ::wasm::SIMDReplaceOp::ReplaceLaneVecF64x2;
2983 : : else if constexpr (sizeof(T) == 4)
2984 : : return ::wasm::SIMDReplaceOp::ReplaceLaneVecF32x4;
2985 : : }
2986 : : M_unreachable("unsupported operation");
2987 : : }();
2988 : 2 : PrimitiveExpr<T, 1> value(primitive_expr_t<U>(std::forward<U>(_value)));
2989 : : auto replacement =
2990 [ + - # # : 4 : M_CONSTEXPR_COND(boolean<T>, (PrimitiveExpr<T, 1>((-value.template to<uint8_t, 1>()).move())), value);
# # # # #
# ]
2991 : 2 : auto referenced_bits = this->referenced_bits(); // moved
2992 : 2 : referenced_bits.splice(referenced_bits.end(), replacement.referenced_bits());
2993 [ - + # # ]: 2 : return PrimitiveExpr(
2994 [ + - + - : 2 : /* expr= */ Module::Builder().makeSIMDReplace(op, this->expr(), M, replacement.expr()),
+ - + - #
# # # # #
# # ]
2995 : 2 : /* referenced_bits= */ std::move(referenced_bits)
2996 : : );
2997 : 2 : }
2998 : :
2999 : : /** Selects lanes of `this` in byte granularity depending on the indices specified by \p indices. Indices `i` in
3000 : : * the range [0, 15] select the `i`-th` lane, indices outside of this range result in the value 0. */
3001 : 42 : PrimitiveExpr swizzle_bytes(PrimitiveExpr<uint8_t, 16> indices) requires (L > 1) {
3002 : 42 : auto referenced_bits = this->referenced_bits(); // moved
3003 [ # # ]: 42 : referenced_bits.splice(referenced_bits.end(), indices.referenced_bits());
3004 [ - + # # : 42 : return PrimitiveExpr(
# # - + -
+ - + # #
# # ]
3005 [ + - + - : 84 : /* expr= */ Module::Builder().makeBinary(::wasm::BinaryOp::SwizzleVecI8x16,
# # # # #
# # # + -
+ - + - +
- + - + -
# # # # #
# # # ]
3006 [ + - # # : 42 : this->expr(),
# # + - +
- + - # #
# # ]
3007 [ + - # # : 42 : indices.expr()),
# # + - +
- + - # #
# # ]
3008 : 42 : /* referenced_bits= */ std::move(referenced_bits)
3009 : : );
3010 : 42 : }
3011 : : /** Selects lanes of `this` in byte granularity depending on the indices specified by \p indices. Indices `i` in
3012 : : * the range [0, L * sizeof(T)) select the `i`-th` lane, indices outside of this range result in the value 0. */
3013 : : template<std::size_t M>
3014 : : requires (M > 0) and (M <= 16) and (M % sizeof(T) == 0)
3015 : 38 : PrimitiveExpr<T, M / sizeof(T)> swizzle_bytes(const std::array<uint8_t, M> &_indices) requires (L > 1) {
3016 : : std::array<uint8_t, 16> indices;
3017 [ # # + + : 162 : for (std::size_t idx = 0; idx < M; ++idx)
+ + # # +
+ + + + +
# # # # #
# # # ]
3018 [ # # + - : 124 : indices[idx] = _indices[idx] < L * sizeof(T) ? _indices[idx] : 16;
+ - # # +
+ + + + -
# # # # #
# # # ]
3019 : 38 : std::fill(indices.begin() + M, indices.end(), 16);
3020 [ # # + - : 38 : PrimitiveExpr<uint8_t, 16> indices_expr(
+ - # # +
- + - + -
# # # # #
# # # ]
3021 [ # # + - : 38 : /* expr= */ Module::Builder().makeConst(::wasm::Literal(indices.data()))
+ - # # +
- + - + -
# # # # #
# # # ]
3022 : : );
3023 [ # # # # : 38 : auto vec = PrimitiveExpr<T, M / sizeof(T)>(swizzle_bytes(indices_expr).move());
# # # # +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3024 [ # # - + : 76 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
- + # # -
+ - + - +
# # # # #
# # # ]
3025 : : vec.template extract_unsafe<0>(), // extract a single value from vector to scalar
3026 : : vec);
3027 : 38 : }
3028 : :
3029 : : /** Selects lanes of `this` in lane granularity depending on the indices specified by \p indices. Indices `i` in
3030 : : * the range [0, L) select the `i`-th` lane, indices outside of this range result in the value 0. */
3031 : : template<std::size_t M>
3032 : : requires (M > 0) and (is_pow_2(M)) and (M * sizeof(T) <= 16)
3033 : 4 : PrimitiveExpr<T, M> swizzle_lanes(const std::array<uint8_t, M> &_indices) requires (L > 1) {
3034 : : std::array<uint8_t, 16> indices;
3035 [ + + # # : 24 : for (std::size_t idx = 0; idx < M; ++idx) {
+ + # # ]
3036 [ + + # # : 60 : for (std::size_t byte = 0; byte < sizeof(T); ++byte)
+ + # # ]
3037 [ + + # # : 40 : indices[idx * sizeof(T) + byte] = _indices[idx] < L ? _indices[idx] * sizeof(T) + byte : 16;
+ + # # ]
3038 : 20 : }
3039 : 4 : std::fill(indices.begin() + M * sizeof(T), indices.end(), 16);
3040 [ + - # # : 4 : PrimitiveExpr<uint8_t, 16> indices_expr(
+ - # # ]
3041 [ + - # # : 4 : /* expr= */ Module::Builder().makeConst(::wasm::Literal(indices.data()))
+ - # # ]
3042 : : );
3043 [ + - + - : 4 : auto vec = PrimitiveExpr<T, M>(swizzle_bytes(indices_expr).move());
+ - + - #
# # # # #
# # + - +
- + - + -
# # # # #
# # # ]
3044 [ - + # # : 8 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
- + # # ]
3045 : : vec.template extract_unsafe<0>(), // extract a single value from vector to scalar
3046 : : vec);
3047 : 4 : }
3048 : :
3049 : :
3050 : : /*------------------------------------------------------------------------------------------------------------------
3051 : : * Printing
3052 : : *----------------------------------------------------------------------------------------------------------------*/
3053 : :
3054 : : friend std::ostream & operator<<(std::ostream &out, const PrimitiveExpr &P) {
3055 : : out << "PrimitiveExpr<" << typeid(type).name() << "," << num_simd_lanes << ">: ";
3056 : : if (P.expr_) out << *P.expr_;
3057 : : else out << "None";
3058 : : return out;
3059 : : }
3060 : :
3061 : : void dump(std::ostream &out) const { out << *this << std::endl; }
3062 : : void dump() const { dump(std::cerr); }
3063 : : };
3064 : :
3065 : : /** Specialization of `PrimitiveExpr<T, L>` for primitive type \tparam T and vectorial values not fitting in a single
3066 : : * SIMD vector, i.e. (multiple) double pumping must be used. Represents an expression (AST) evaluating to \tparam L
3067 : : * runtime values of primitive type \tparam T. */
3068 : : template<dsl_primitive T, std::size_t L>
3069 : : requires (L > 1) and (is_pow_2(L)) and (L * sizeof(T) > 16) and ((L * sizeof(T)) % 16 == 0)
3070 : : struct PrimitiveExpr<T, L>
3071 : : {
3072 : : ///> the primitive type of the represented expression
3073 : : using type = T;
3074 : : ///> the number of SIMD lanes of the represented expression
3075 : : static constexpr std::size_t num_simd_lanes = L;
3076 : : ///> the type of a single fully utilized vector
3077 : : using vector_type = PrimitiveExpr<T, 16 / sizeof(T)>;
3078 : : ///> the number of SIMD vectors needed for the represented expression
3079 : : static constexpr std::size_t num_vectors = (L * sizeof(T)) / 16;
3080 : : static_assert(num_vectors >= 2);
3081 : :
3082 : : /*----- Friends --------------------------------------------------------------------------------------------------*/
3083 : : template<typename, std::size_t> friend struct PrimitiveExpr; // to convert U to T and U* to uint32_t
3084 : : template<typename, std::size_t>
3085 : : friend struct Expr; // to construct an empty `PrimitiveExpr<bool>` for the NULL information
3086 : : template<typename, VariableKind, bool, std::size_t>
3087 : : friend class detail::variable_storage; // to construct from `::wasm::Expression` and access private `expr()`
3088 : : friend struct Module; // to access internal `::wasm::Expression`, e.g. in `emit_return()`
3089 : : friend struct Block; // to access internal `::wasm::Expression`, e.g. in `go_to()`
3090 : : template<typename> friend struct FunctionProxy; // to access internal `::wasm::Expr` to construct function calls
3091 : : template<typename> friend struct invoke_interpreter; // to access private `expr()`
3092 : :
3093 : : private:
3094 : : ///> the fully utilized SIMD vectors represented as `PrimitiveExpr`s
3095 : : std::array<vector_type, num_vectors> vectors_;
3096 : :
3097 : : private:
3098 : : ///> Constructs an empty `PrimitiveExpr`, for which `operator bool()` returns `false`.
3099 : 0 : explicit PrimitiveExpr() {
3100 : : /* Do not use `vectors_.fill()` since it internally delegates to `std::fill_n()` which tries to copy-assign the
3101 : : * given value to each slot, however, the assignment operator of `vector_type` is private and thus not
3102 : : * accessible. Therefore, implement this logic by ourselves as we are befriended with `vector_type`. */
3103 [ # # ]: 0 : for (auto it = vectors_.begin(); it != vectors_.end(); ++it)
3104 : 0 : *it = vector_type();
3105 : 0 : }
3106 : :
3107 : : ///> Constructs a `PrimitiveExpr` from an array of fully utilized `PrimitiveExpr`s \p vectors.
3108 : 242 : explicit PrimitiveExpr(std::array<vector_type, num_vectors> vectors) : vectors_(std::move(vectors)) { }
3109 : : ///> Constructs a `PrimitiveExpr` from an initializer list of fully utilized `PrimitiveExpr`s \p vectors.
3110 : : explicit PrimitiveExpr(std::initializer_list<vector_type> vectors) : vectors_(std::move(vectors)) { }
3111 : :
3112 : : public:
3113 : : /** Constructs a new `PrimitiveExpr` from a constant \p value. */
3114 : : template<dsl_primitive... Us>
3115 : : requires (sizeof...(Us) > 0) and
3116 : : requires (Us... us) { { make_literal<T, L>(us...) } -> std::same_as<std::array<::wasm::Literal, num_vectors>>; }
3117 : 50 : explicit PrimitiveExpr(Us... value)
3118 [ + - + - : 100 : : PrimitiveExpr([&](){
# # # # #
# # # + -
+ - # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # ]
3119 : 50 : std::array<vector_type, num_vectors> vectors;
3120 : 50 : auto it = vectors.begin();
3121 [ + - + + : 218 : for (auto literal : make_literal<T, L>(value...))
+ - + - +
+ + - + -
+ + + - #
# # # # #
# # # # #
# # # # #
# # + - +
+ + - + -
+ + + - #
# # # # #
# # # # #
# + - + +
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + +
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ + + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3122 [ + - + - : 168 : *(it++) = vector_type(Module::Builder().makeConst(literal));
+ - + - #
# + - + -
+ - + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3123 [ + - + - : 50 : M_insist(it == vectors.end());
+ - # # #
# # # + -
+ - # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # ]
3124 [ + - + - : 50 : return std::move(vectors);
+ - # # #
# # # + -
+ - # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # ]
3125 : 50 : }())
3126 : 50 : { }
3127 : :
3128 : : /** Constructs a new `PrimitiveExpr` from a decayable constant \p value. */
3129 : : template<decayable... Us>
3130 : : requires (sizeof...(Us) > 0) and (dsl_primitive<std::decay_t<Us>> and ...) and
3131 : : requires (Us... us) { PrimitiveExpr(std::decay_t<Us>(us)...); }
3132 : : explicit PrimitiveExpr(Us... value)
3133 : : : PrimitiveExpr(std::decay_t<Us>(value)...)
3134 : : { }
3135 : :
3136 : : PrimitiveExpr(const PrimitiveExpr&) = delete;
3137 : : /** Constructs a new `PrimitiveExpr` by **moving** the underlying `vectors_` of `other` to `this`. */
3138 [ # # # # : 142 : PrimitiveExpr(PrimitiveExpr &other) : PrimitiveExpr(std::move(other.vectors_)) { /* move, not copy */ }
# # # # #
# + - # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - #
# # # # #
# # ]
3139 : : /** Constructs a new `PrimitiveExpr` by **moving** the underlying `vectors_` of `other` to `this`. */
3140 [ + - # # : 4 : PrimitiveExpr(PrimitiveExpr &&other) : PrimitiveExpr(std::move(other.vectors_)) { }
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3141 : :
3142 : : PrimitiveExpr & operator=(PrimitiveExpr&&) = delete;
3143 : :
3144 : 242 : ~PrimitiveExpr() = default;
3145 : :
3146 : : private:
3147 : : /** **Moves** the underlying `PrimitiveExpr<T, 16 / sizeof(T)>`s out of `this`. */
3148 : 16 : std::array<vector_type, num_vectors> vectors() { return std::move(vectors_); }
3149 : : /** **Moves** the underlying vectors as `PrimitiveExpr<U, 16 / sizeof(U)>` out of `this`. */
3150 : : template<dsl_primitive U, std::size_t M>
3151 : : requires ((M * sizeof(U)) / 16 == num_vectors)
3152 : 0 : auto move() {
3153 : : using ToVecT = PrimitiveExpr<U, 16 / sizeof(U)>;
3154 : 0 : std::array<ToVecT, num_vectors> vectors;
3155 [ # # # # : 0 : for (std::size_t idx = 0; idx < num_vectors; ++idx)
# # # # #
# ]
3156 [ # # # # : 0 : vectors[idx] = ToVecT(vectors_[idx].move());
# # # # #
# # # # #
# # # # #
# ]
3157 [ # # # # : 0 : return std::move(vectors);
# # # # #
# ]
3158 : 0 : }
3159 : :
3160 : : public:
3161 : : /** Returns `true` if this `PrimitiveExpr` actually holds a value (Binaryen AST), `false` otherwise. Can be used to
3162 : : * test whether this `PrimitiveExpr` has already been used. */
3163 : 116 : explicit operator bool() const {
3164 : 414 : return std::all_of(vectors_.cbegin(), vectors_.cend(), [](const auto &expr){ return bool(expr); });
3165 : : }
3166 : :
3167 : : /** Creates and returns a *deep copy* of `this`. */
3168 : 0 : PrimitiveExpr clone() const {
3169 : 0 : M_insist(bool(*this), "cannot clone an already moved or discarded `PrimitiveExpr`");
3170 : 0 : std::array<vector_type, num_vectors> vectors_cpy;
3171 [ # # # # : 0 : for (std::size_t idx = 0; idx < num_vectors; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3172 [ # # # # : 0 : vectors_cpy[idx] = vectors_[idx].clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3173 [ # # # # : 0 : return PrimitiveExpr(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3174 [ # # # # : 0 : /* vectors= */ std::move(vectors_cpy)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3175 : : );
3176 : 0 : }
3177 : :
3178 : : /** Discards `this`. This is necessary to signal in our DSL that a value is *expectedly* unused (and not dead
3179 : : * code). For example, the return value of a function that was invoked because of its side effects may remain
3180 : : * unused. One **must** discard the returned value to signal that the value is expectedly left unused. */
3181 : 4 : void discard() {
3182 : 4 : M_insist(bool(*this), "cannot discard an already moved or discarded `PrimitiveExpr`");
3183 : 12 : std::for_each(vectors_.begin(), vectors_.end(), [](auto &expr){ expr.discard(); });
3184 : 4 : }
3185 : :
3186 : :
3187 : : /*------------------------------------------------------------------------------------------------------------------
3188 : : * Conversion operations
3189 : : *----------------------------------------------------------------------------------------------------------------*/
3190 : :
3191 : : private:
3192 : : template<dsl_primitive U, std::size_t M>
3193 : : requires ((M * sizeof(U)) % 16 == 0)
3194 : 102 : PrimitiveExpr<U, M> convert() {
3195 : : using From = T;
3196 : : using To = U;
3197 : 102 : constexpr std::size_t FromL = L;
3198 : 102 : constexpr std::size_t ToL = M;
3199 : : using FromVecT = PrimitiveExpr<T, 16 / sizeof(T)>;
3200 : : using ToVecT = PrimitiveExpr<U, 16 / sizeof(U)>;
3201 : 102 : constexpr std::size_t FromVecL = (L * sizeof(T)) / 16;
3202 : 102 : constexpr std::size_t ToVecL = (M * sizeof(U)) / 16;
3203 : :
3204 : : if constexpr (std::same_as<From, To> and FromL == ToL)
3205 : 32 : return *this;
3206 : : if constexpr (integral<From> and integral<To> and std::is_signed_v<From> == std::is_signed_v<To> and
3207 : : sizeof(From) == sizeof(To) and FromL == ToL)
3208 : : return PrimitiveExpr<To, ToL>(move<To, ToL>());
3209 : :
3210 : : if constexpr (boolean<From>) { // from boolean
3211 : : if constexpr (integral<To>) { // to integer
3212 : : if constexpr (FromL == ToL) { // vectorial
3213 : : if constexpr (sizeof(To) == 1) // bool -> i8/u8
3214 [ # # # # : 0 : return -PrimitiveExpr<To, ToL>(move<To, ToL>()); // negate to convert 0xff to 1
# # # # ]
3215 : : if constexpr (std::is_signed_v<To>) {
3216 : : if constexpr (sizeof(To) == 2) // bool -> i16
3217 [ # # ]: 0 : return to<int8_t, ToL>().template to<To, ToL>();
3218 : : if constexpr (sizeof(To) == 4) // bool -> i32
3219 [ # # ]: 0 : return to<int8_t, ToL>().template to<To, ToL>();
3220 : : if constexpr (sizeof(To) == 8) // bool -> i64
3221 [ # # ]: 0 : return to<int8_t, ToL>().template to<To, ToL>();
3222 : : } else {
3223 : : if constexpr (sizeof(To) == 2) // bool -> u16
3224 [ # # ]: 0 : return to<uint8_t, ToL>().template to<To, ToL>();
3225 : : if constexpr (sizeof(To) == 4) // bool -> u32
3226 [ # # ]: 0 : return to<uint8_t, ToL>().template to<To, ToL>();
3227 : : if constexpr (sizeof(To) == 8) // bool -> u64
3228 [ # # ]: 0 : return to<uint8_t, ToL>().template to<To, ToL>();
3229 : : }
3230 : : }
3231 : : }
3232 : : if constexpr (std::floating_point<To>) { // to floating point
3233 : : if constexpr (FromL == ToL) { // vectorial
3234 : : if constexpr (sizeof(To) == 4) // bool -> f32
3235 [ # # ]: 0 : return to<uint32_t, ToL>().template convert<To, ToL>();
3236 : : if constexpr (sizeof(To) == 8) // bool -> f64
3237 [ # # ]: 0 : return to<uint32_t, ToL>().template convert<To, ToL>();
3238 : : }
3239 : : }
3240 : : }
3241 : :
3242 : : if constexpr (boolean<To>) { // to boolean
3243 : : if constexpr (integral<From>) { // from integer
3244 : : if constexpr (FromL == ToL) { // vectorial
3245 : : if constexpr (sizeof(From) == 1) // i8/u8 -> bool
3246 [ # # ]: 0 : return *this != PrimitiveExpr(static_cast<From>(0));
3247 : : if constexpr (std::is_signed_v<From>) {
3248 : : if constexpr (sizeof(From) == 2) // i16 -> bool
3249 : : return to<int8_t, ToL>().template to<To, ToL>();
3250 : : if constexpr (sizeof(From) == 4 and FromVecL >= 4) // i32 -> bool
3251 : : return to<int8_t, ToL>().template to<To, ToL>();
3252 : : if constexpr (sizeof(From) == 8 and FromVecL >= 8) // i64 -> bool
3253 : : return to<int8_t, ToL>().template to<To, ToL>();
3254 : : } else {
3255 : : if constexpr (sizeof(From) == 2) // u16 -> bool
3256 [ # # # # ]: 0 : return to<uint8_t, ToL>().template to<To, ToL>();
3257 : : if constexpr (sizeof(From) == 4 and FromVecL >= 4) // u32 -> bool
3258 [ # # # # ]: 0 : return to<uint8_t, ToL>().template to<To, ToL>();
3259 : : if constexpr (sizeof(From) == 8 and FromVecL >= 8) // u64 -> bool
3260 [ # # # # ]: 0 : return to<uint8_t, ToL>().template to<To, ToL>();
3261 : : }
3262 : : }
3263 : : }
3264 : : if constexpr (std::floating_point<From>) { // from floating point
3265 : : if constexpr (FromL == ToL) { // vectorial
3266 : : if constexpr (sizeof(From) == 4 and FromVecL >= 4) // f32 -> bool
3267 : : return *this != PrimitiveExpr(static_cast<From>(0));
3268 : : if constexpr (sizeof(From) == 8 and FromVecL >= 8) // f64 -> bool
3269 : : return *this != PrimitiveExpr(static_cast<From>(0));
3270 : : }
3271 : : }
3272 : : }
3273 : :
3274 : : if constexpr (integral<From>) { // from integer
3275 : : if constexpr (integral<To>) { // to integer
3276 : : if constexpr (FromL == ToL) { // vectorial
3277 : : if constexpr (std::is_signed_v<From>) { // signed
3278 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 2) { // i8 -> i16
3279 : 0 : std::array<ToVecT, ToVecL> vectors;
3280 [ # # ]: 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
3281 [ # # # # ]: 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
3282 : : ::wasm::ExtendLowSVecI8x16ToVecI16x8
3283 : : );
3284 [ # # ]: 0 : vectors[2 * idx + 1] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
3285 : : ::wasm::ExtendHighSVecI8x16ToVecI16x8
3286 : : );
3287 : 0 : }
3288 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
3289 : 0 : }
3290 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 1) { // i16 -> i8
3291 : 6 : std::array<ToVecT, ToVecL> vectors;
3292 [ + + # # ]: 12 : for (std::size_t idx = 0; idx < ToVecL; ++idx)
3293 [ + - # # ]: 6 : vectors[idx] = vectors_[2 * idx].template binary<To, ToVecT::num_simd_lanes>(
3294 [ + - # # ]: 6 : ::wasm::NarrowSVecI16x8ToVecI8x16, vectors_[2 * idx + 1]
3295 : : );
3296 [ + - + - : 6 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3297 : 6 : }
3298 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // i8 -> i32
3299 [ # # ]: 0 : return to<int16_t, ToL>().template to<To, ToL>();
3300 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 1 and FromVecL >= 4) // i32 -> i8
3301 [ + - # # ]: 6 : return to<int16_t, ToL>().template to<To, ToL>();
3302 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // i8 -> i64
3303 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3304 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 1 and FromVecL >= 8) // i64 -> i8
3305 [ # # # # ]: 0 : return to<int16_t, ToL>().template to<To, ToL>();
3306 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4) { // i16 -> i32
3307 : 0 : std::array<ToVecT, ToVecL> vectors;
3308 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
3309 [ # # # # : 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
# # # # ]
3310 : : ::wasm::ExtendLowSVecI16x8ToVecI32x4
3311 : : );
3312 [ # # # # ]: 0 : vectors[2 * idx + 1] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
3313 : : ::wasm::ExtendHighSVecI16x8ToVecI32x4
3314 : : );
3315 : 0 : }
3316 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3317 : 0 : }
3318 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 2) { // i32 -> i16
3319 : 6 : std::array<ToVecT, ToVecL> vectors;
3320 [ # # + + : 18 : for (std::size_t idx = 0; idx < ToVecL; ++idx)
# # ]
3321 [ # # + - : 12 : vectors[idx] = vectors_[2 * idx].template binary<To, ToVecT::num_simd_lanes>(
# # ]
3322 [ # # + - : 12 : ::wasm::NarrowSVecI32x4ToVecI16x8, vectors_[2 * idx + 1]
# # ]
3323 : : );
3324 [ # # # # : 6 : return PrimitiveExpr<To, ToL>(std::move(vectors));
+ - + - #
# # # ]
3325 : 6 : }
3326 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // i16 -> i64
3327 [ # # # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3328 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 2 and FromVecL >= 4) // i64 -> i16
3329 [ # # # # : 0 : return to<int32_t, ToL>().template to<To, ToL>();
# # ]
3330 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) { // i32 -> i64
3331 : 0 : std::array<ToVecT, ToVecL> vectors;
3332 [ # # # # : 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
# # ]
3333 [ # # # # : 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
# # # # #
# # # ]
3334 : : ::wasm::ExtendLowSVecI32x4ToVecI64x2
3335 : : );
3336 [ # # # # : 0 : vectors[2 * idx + 1] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
# # ]
3337 : : ::wasm::ExtendHighSVecI32x4ToVecI64x2
3338 : : );
3339 : 0 : }
3340 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # ]
3341 : 0 : }
3342 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) { // i64 -> i32
3343 : 4 : std::array<ToVecT, ToVecL> vectors;
3344 [ + + # # : 8 : for (std::size_t idx = 0; idx < ToVecL; ++idx) {
# # # # ]
3345 : 4 : std::array<uint8_t, 16> indices =
3346 : : { 0, 1, 2, 3, 8, 9 , 10, 11, 16, 17, 18, 19, 24, 25, 26, 27 };
3347 : 4 : vectors[idx] =
3348 [ + - + - : 4 : ToVecT(ShuffleBytes(vectors_[2 * idx], vectors_[2 * idx + 1], indices).move());
- + # # #
# # # # #
# # # # #
# # # #
# ]
3349 : 4 : }
3350 [ + - - + : 4 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # # #
# # ]
3351 : 4 : }
3352 : : } else { // unsigned
3353 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 2) { // u8 -> u16
3354 : 0 : std::array<ToVecT, ToVecL> vectors;
3355 [ # # ]: 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
3356 [ # # # # ]: 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
3357 : : ::wasm::ExtendLowUVecI8x16ToVecI16x8
3358 : : );
3359 [ # # ]: 0 : vectors[2 * idx + 1] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
3360 : : ::wasm::ExtendHighUVecI8x16ToVecI16x8
3361 : : );
3362 : 0 : }
3363 [ # # # # ]: 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
3364 : 0 : }
3365 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 1) { // u16 -> u8
3366 : 16 : std::array<ToVecT, ToVecL> vectors;
3367 [ + + # # ]: 32 : for (std::size_t idx = 0; idx < ToVecL; ++idx)
3368 [ + - - + : 16 : vectors[idx] = vectors_[2 * idx].template binary<To, ToVecT::num_simd_lanes>(
# # ]
3369 [ + - # # ]: 16 : ::wasm::NarrowSVecI16x8ToVecI8x16, vectors_[2 * idx + 1]
3370 : : );
3371 [ + - - + : 16 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3372 : 16 : }
3373 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // u8 -> u32
3374 [ # # ]: 0 : return to<uint16_t, ToL>().template to<To, ToL>();
3375 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 1 and FromVecL >= 4) // u32 -> u8
3376 [ + - # # ]: 16 : return to<uint16_t, ToL>().template to<To, ToL>();
3377 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // u8 -> u64
3378 [ # # ]: 0 : return to<uint32_t, ToL>().template to<To, ToL>();
3379 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 1 and FromVecL >= 8) // u64 -> u8
3380 [ # # # # ]: 0 : return to<uint16_t, ToL>().template to<To, ToL>();
3381 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4) { // u16 -> u32
3382 : 0 : std::array<ToVecT, ToVecL> vectors;
3383 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
3384 [ # # # # : 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
# # # # ]
3385 : : ::wasm::ExtendLowUVecI16x8ToVecI32x4
3386 : : );
3387 [ # # # # ]: 0 : vectors[2 * idx + 1] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
3388 : : ::wasm::ExtendHighUVecI16x8ToVecI32x4
3389 : : );
3390 : 0 : }
3391 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3392 : 0 : }
3393 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 2) { // u32 -> u16
3394 : 16 : std::array<ToVecT, ToVecL> vectors;
3395 [ + + # # ]: 48 : for (std::size_t idx = 0; idx < ToVecL; ++idx)
3396 [ + - - + : 32 : vectors[idx] = vectors_[2 * idx].template binary<To, ToVecT::num_simd_lanes>(
# # ]
3397 [ + - # # ]: 32 : ::wasm::NarrowSVecI32x4ToVecI16x8, vectors_[2 * idx + 1]
3398 : : );
3399 [ + - - + : 16 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3400 : 16 : }
3401 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // u16 -> u64
3402 : : return to<uint32_t, ToL>().template to<To, ToL>();
3403 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 2 and FromVecL >= 4) // u64 -> u16
3404 [ # # # # ]: 0 : return to<uint32_t, ToL>().template to<To, ToL>();
3405 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) { // u32 -> u64
3406 : 0 : std::array<ToVecT, ToVecL> vectors;
3407 [ # # # # : 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
# # ]
3408 [ # # # # : 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
# # # # #
# # # ]
3409 : : ::wasm::ExtendLowUVecI32x4ToVecI64x2
3410 : : );
3411 [ # # # # : 0 : vectors[2 * idx + 1] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
# # ]
3412 : : ::wasm::ExtendHighUVecI32x4ToVecI64x2
3413 : : );
3414 : 0 : }
3415 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # ]
3416 : 0 : }
3417 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) { // u64 -> u32
3418 : 0 : std::array<ToVecT, ToVecL> vectors;
3419 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < ToVecL; ++idx) {
3420 : 0 : std::array<uint8_t, 16> indices =
3421 : : { 0, 1, 2, 3, 8, 9 , 10, 11, 16, 17, 18, 19, 24, 25, 26, 27 };
3422 [ # # ]: 0 : vectors[idx] =
3423 [ # # # # : 0 : ToVecT(ShuffleBytes(vectors_[2 * idx], vectors_[2 * idx + 1], indices).move());
# # # # #
# # # ]
3424 : 0 : }
3425 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3426 : 0 : }
3427 : : }
3428 : : }
3429 : : }
3430 : : if constexpr (std::floating_point<To>) { // to floating point
3431 : : if constexpr (FromL == ToL) { // vectorial
3432 : : if constexpr (std::is_signed_v<From>) { // signed
3433 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // i8 -> f32
3434 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3435 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // i8 -> f64
3436 [ # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3437 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4) // i16 -> f32
3438 [ # # # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3439 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // i16 -> f64
3440 [ # # # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3441 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) { // i32 -> f32
3442 : 0 : std::array<ToVecT, ToVecL> vectors;
3443 [ # # # # : 0 : for (std::size_t idx = 0; idx < ToVecL; ++idx)
# # ]
3444 [ # # # # : 0 : vectors[idx] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
# # ]
3445 : : ::wasm::ConvertSVecI32x4ToVecF32x4
3446 : : );
3447 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # ]
3448 : 0 : }
3449 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) { // i32 -> f64
3450 : 0 : std::array<ToVecT, ToVecL> vectors;
3451 [ # # # # : 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
# # ]
3452 [ # # # # : 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
# # # # #
# # # ]
3453 : : ::wasm::ConvertLowSVecI32x4ToVecF64x2
3454 : : );
3455 [ # # # # : 0 : auto high_to_low = vectors_[idx].swizzle_lanes(std::to_array<uint8_t>({ 2, 3 }));
# # ]
3456 [ # # # # : 0 : vectors[2 * idx + 1] = high_to_low.template unary<To, ToVecT::num_simd_lanes>(
# # ]
3457 : : ::wasm::ConvertLowSVecI32x4ToVecF64x2
3458 : : );
3459 : 0 : }
3460 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # ]
3461 : 0 : }
3462 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // i64 -> f32
3463 [ # # # # : 0 : return to<int32_t, ToL>().template to<To, ToL>();
# # # # ]
3464 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // i64 -> f64
3465 [ # # # # : 0 : return to<int32_t, ToL>().template to<To, ToL>();
# # # # ]
3466 : : } else { // unsigned
3467 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 4) // u8 -> f32
3468 : : return to<uint32_t, ToL>().template convert<To, ToL>();
3469 : : if constexpr (sizeof(From) == 1 and sizeof(To) == 8) // u8 -> f64
3470 : : return to<uint32_t, ToL>().template convert<To, ToL>();
3471 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 4) // u16 -> f32
3472 : : return to<uint32_t, ToL>().template convert<To, ToL>();
3473 : : if constexpr (sizeof(From) == 2 and sizeof(To) == 8) // u16 -> f64
3474 : : return to<uint32_t, ToL>().template convert<To, ToL>();
3475 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) { // u32 -> f32
3476 : 0 : std::array<ToVecT, ToVecL> vectors;
3477 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < ToVecL; ++idx)
3478 [ # # # # ]: 0 : vectors[idx] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
3479 : : ::wasm::ConvertUVecI32x4ToVecF32x4
3480 : : );
3481 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3482 : 0 : }
3483 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) { // u32 -> f64
3484 : 0 : std::array<ToVecT, ToVecL> vectors;
3485 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
3486 [ # # # # : 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
# # # # ]
3487 : : ::wasm::ConvertLowUVecI32x4ToVecF64x2
3488 : : );
3489 [ # # # # ]: 0 : auto high_to_low = vectors_[idx].swizzle_lanes(std::to_array<uint8_t>({ 2, 3 }));
3490 [ # # # # ]: 0 : vectors[2 * idx + 1] = high_to_low.template unary<To, ToVecT::num_simd_lanes>(
3491 : : ::wasm::ConvertLowUVecI32x4ToVecF64x2
3492 : : );
3493 : 0 : }
3494 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # ]
3495 : 0 : }
3496 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) // u64 -> f32
3497 : : return to<uint32_t, ToL>().template convert<To, ToL>();
3498 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // u64 -> f64
3499 : : return to<uint32_t, ToL>().template convert<To, ToL>();
3500 : : }
3501 : : }
3502 : : }
3503 : : }
3504 : :
3505 : : if constexpr (std::floating_point<From>) { // from floating point
3506 : : if constexpr (integral<To>) { // to integer
3507 : : if constexpr (FromL == ToL) { // vectorial
3508 : : if constexpr (std::is_signed_v<To>) { // signed
3509 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 1 and FromVecL >= 4) // f32 -> i8
3510 [ # # # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3511 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 1 and FromVecL >= 8) // f64 -> i8
3512 [ # # # # ]: 0 : return to<int32_t, ToL>().template to<To, ToL>();
3513 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 2) // f32 -> i16
3514 [ # # # # : 0 : return to<int32_t, ToL>().template to<To, ToL>();
# # ]
3515 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 2 and FromVecL >= 4) // f64 -> i16
3516 [ # # # # : 0 : return to<int32_t, ToL>().template to<To, ToL>();
# # ]
3517 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) { // f32 -> i32
3518 : 0 : std::array<ToVecT, ToVecL> vectors;
3519 [ # # # # : 0 : for (std::size_t idx = 0; idx < ToVecL; ++idx)
# # ]
3520 [ # # # # : 0 : vectors[idx] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
# # ]
3521 : : ::wasm::TruncSatSVecF32x4ToVecI32x4
3522 : : );
3523 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # ]
3524 : 0 : }
3525 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) { // f64 -> i32
3526 : 0 : std::array<ToVecT, ToVecL> vectors;
3527 [ # # # # : 0 : for (std::size_t idx = 0; idx < ToVecL; ++idx) {
# # # # ]
3528 [ # # # # : 0 : auto low = vectors_[2 * idx].template unary<To, ToVecT::num_simd_lanes>(
# # # # ]
3529 : : ::wasm::TruncSatZeroSVecF64x2ToVecI32x4
3530 : : );
3531 [ # # # # : 0 : auto high = vectors_[2 * idx + 1].template unary<To, ToVecT::num_simd_lanes>(
# # # # ]
3532 : : ::wasm::TruncSatZeroSVecF64x2ToVecI32x4
3533 : : );
3534 [ # # # # : 0 : vectors[idx] = ShuffleLanes(low, high, std::to_array<uint8_t>({ 0, 1, 4, 5 }));
# # # # ]
3535 : 0 : }
3536 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # # #
# # ]
3537 : 0 : }
3538 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) // f32 -> i64
3539 [ # # # # : 0 : return to<int32_t, ToL>().template to<To, ToL>();
# # ]
3540 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // f64 -> i64
3541 [ # # # # : 0 : return to<int32_t, ToL>().template to<To, ToL>();
# # # # ]
3542 : : } else { // unsigned
3543 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 1 and FromVecL >= 4) // f32 -> u8
3544 : : return convert<uint32_t, ToL>().template to<To, ToL>();
3545 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 1 and FromVecL >= 8) // f64 -> u8
3546 : : return convert<uint32_t, ToL>().template to<To, ToL>();
3547 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 2) // f32 -> u16
3548 : : return convert<uint32_t, ToL>().template to<To, ToL>();
3549 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 2 and FromVecL >= 4) // f64 -> u16
3550 : : return convert<uint32_t, ToL>().template to<To, ToL>();
3551 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 4) { // f32 -> u32
3552 : : std::array<ToVecT, ToVecL> vectors;
3553 : : for (std::size_t idx = 0; idx < ToVecL; ++idx)
3554 : : vectors[idx] = vectors_[idx].template unary<To, ToVecT::num_simd_lanes>(
3555 : : ::wasm::TruncSatUVecF32x4ToVecI32x4
3556 : : );
3557 : : return PrimitiveExpr<To, ToL>(std::move(vectors));
3558 : : }
3559 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) { // f64 -> u32
3560 : : std::array<ToVecT, ToVecL> vectors;
3561 : : for (std::size_t idx = 0; idx < ToVecL; ++idx) {
3562 : : auto low = vectors_[2 * idx].template unary<To, ToVecT::num_simd_lanes>(
3563 : : ::wasm::TruncSatZeroUVecF64x2ToVecI32x4
3564 : : );
3565 : : auto high = vectors_[2 * idx + 1].template unary<To, ToVecT::num_simd_lanes>(
3566 : : ::wasm::TruncSatZeroUVecF64x2ToVecI32x4
3567 : : );
3568 : : vectors[idx] = ShuffleLanes(low, high, std::to_array<uint8_t>({ 0, 1, 4, 5 }));
3569 : : }
3570 : : return PrimitiveExpr<To, ToL>(std::move(vectors));
3571 : : }
3572 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) // f32 -> u64
3573 : : return convert<uint32_t, ToL>().template to<To, ToL>();
3574 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 8) // f64 -> u64
3575 : : return convert<uint32_t, ToL>().template to<To, ToL>();
3576 : : }
3577 : : }
3578 : : }
3579 : : if constexpr (std::floating_point<To>) { // to floating point
3580 : : if constexpr (FromL == ToL) { // vectorial
3581 : : if constexpr (sizeof(From) == 4 and sizeof(To) == 8) { // f32 -> f64
3582 : 0 : std::array<ToVecT, ToVecL> vectors;
3583 [ # # # # : 0 : for (std::size_t idx = 0; idx < FromVecL; ++idx) {
# # ]
3584 [ # # # # : 0 : vectors[2 * idx] = vectors_[idx].clone().template unary<To, ToVecT::num_simd_lanes>(
# # # # #
# # # ]
3585 : : ::wasm::PromoteLowVecF32x4ToVecF64x2
3586 : : );
3587 [ # # # # : 0 : auto high_to_low = vectors_[idx].swizzle_lanes(std::to_array<uint8_t>({ 2, 3 }));
# # ]
3588 [ # # # # : 0 : vectors[2 * idx + 1] = high_to_low.template unary<To, ToVecT::num_simd_lanes>(
# # ]
3589 : : ::wasm::PromoteLowVecF32x4ToVecF64x2
3590 : : );
3591 : 0 : }
3592 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # ]
3593 : 0 : }
3594 : : if constexpr (sizeof(From) == 8 and sizeof(To) == 4) { // f64 -> f32
3595 : 0 : std::array<ToVecT, ToVecL> vectors;
3596 [ # # # # : 0 : for (std::size_t idx = 0; idx < ToVecL; ++idx) {
# # # # ]
3597 [ # # # # : 0 : auto low = vectors_[2 * idx].template unary<To, ToVecT::num_simd_lanes>(
# # # # ]
3598 : : ::wasm::DemoteZeroVecF64x2ToVecF32x4
3599 : : );
3600 [ # # # # : 0 : auto high = vectors_[2 * idx + 1].template unary<To, ToVecT::num_simd_lanes>(
# # # # ]
3601 : : ::wasm::DemoteZeroVecF64x2ToVecF32x4
3602 : : );
3603 [ # # # # : 0 : vectors[idx] = ShuffleLanes(low, high, std::to_array<uint8_t>({ 0, 1, 4, 5 }));
# # # # ]
3604 : 0 : }
3605 [ # # # # : 0 : return PrimitiveExpr<To, ToL>(std::move(vectors));
# # # # #
# # # # #
# # ]
3606 : 0 : }
3607 : : }
3608 : : }
3609 : : }
3610 : :
3611 : : M_unreachable("illegal conversion");
3612 : 48 : }
3613 : :
3614 : : public:
3615 : : /** Implicit conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<To, ToL>`. Only applicable if
3616 : : *
3617 : : * - `L` and `ToL` are equal, i.e. conversion does not change the number of SIMD lanes
3618 : : * - `T` and `To` have same signedness
3619 : : * - neither or both `T` and `To` are integers
3620 : : * - `T` can be *trivially* converted to `To` (e.g. `int` to `long` but not `long` to `int`)
3621 : : * - `To` is not `bool`
3622 : : */
3623 : : template<dsl_primitive To, std::size_t ToL = L>
3624 : : requires (L == ToL) and // L and ToL are equal
3625 : : same_signedness<T, To> and // T and To have same signedness
3626 : : (integral<T> == integral<To>) and // neither nor both T and To are integers (excluding bool)
3627 : : (sizeof(T) <= sizeof(To)) // T can be *trivially* converted to To
3628 : : operator PrimitiveExpr<To, ToL>() { return convert<To, ToL>(); }
3629 : :
3630 : : /** Explicit conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<To, ToL>`. Only applicable if
3631 : : *
3632 : : * - `L` and `ToL` are equal, i.e. conversion does not change the number of SIMD lanes
3633 : : * - `T` and `To` have same signedness or `T` is `bool` or `char` or `To` is `bool` or `char
3634 : : * - `T` can be converted to `To` (e.g. `int` to `long`, `long` to `int`, `float` to `int`)
3635 : : * - narrowing `T` to `To` results in at least one fully utilized SIMD vector
3636 : : */
3637 : : template<dsl_primitive To, std::size_t ToL = L>
3638 : : requires (L == ToL) and // L and ToL are equal
3639 : : (same_signedness<T, To> or // T and To have same signedness
3640 : : boolean<T> or std::same_as<T, char> or // or T is bool or char
3641 : : boolean<To> or std::same_as<To, char>) and // or To is bool or char
3642 : : std::is_convertible_v<T, To> and // T can be converted to To
3643 : : (num_vectors >= sizeof(T) / sizeof(To)) // at least one vector afterwards
3644 : 102 : PrimitiveExpr<To, ToL> to() { return convert<To, ToL>(); }
3645 : :
3646 : : /** Conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<std::make_signed_t<T>, L>`. Only applicable if
3647 : : *
3648 : : * - `T` is an unsigned integral type except `bool`
3649 : : */
3650 : : auto make_signed() requires unsigned_integral<T> {
3651 : : return PrimitiveExpr<std::make_signed_t<T>, L>(move<std::make_signed_t<T>, L>());
3652 : : }
3653 : :
3654 : : /** Conversion of a `PrimitiveExpr<T, L>` to a `PrimitiveExpr<std::make_unsigned_t<T>, L>`. Only available if
3655 : : *
3656 : : * - `T` is a signed integral type except `bool`
3657 : : */
3658 : : auto make_unsigned() requires signed_integral<T> {
3659 : : return PrimitiveExpr<std::make_unsigned_t<T>, L>(move<std::make_unsigned_t<T>, L>());
3660 : : }
3661 : :
3662 : :
3663 : : /*------------------------------------------------------------------------------------------------------------------
3664 : : * Unary operations
3665 : : *----------------------------------------------------------------------------------------------------------------*/
3666 : :
3667 : : #define UNARY(OP) \
3668 : : auto OP() requires requires (vector_type v) { v.OP(); } { \
3669 : : using ResVecT = decltype(std::declval<vector_type>().OP()); \
3670 : : static_assert(ResVecT::num_simd_lanes * sizeof(typename ResVecT::type) == 16, \
3671 : : "result vectors must be fully utilized"); \
3672 : : std::array<ResVecT, num_vectors> vectors; \
3673 : : for (std::size_t idx = 0; idx < num_vectors; ++idx) \
3674 : : vectors[idx] = vectors_[idx].OP(); \
3675 : : return PrimitiveExpr<typename ResVecT::type, ResVecT::num_simd_lanes * num_vectors>(std::move(vectors)); \
3676 : : }
3677 : :
3678 [ # # # # : 0 : UNARY(operator +)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3679 [ # # # # : 0 : UNARY(operator -)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3680 : : UNARY(abs)
3681 : : UNARY(ceil)
3682 : : UNARY(floor)
3683 : : UNARY(trunc)
3684 : : UNARY(nearest)
3685 : : UNARY(sqrt)
3686 : : UNARY(add_pairwise)
3687 [ # # # # : 0 : UNARY(operator ~)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3688 : : UNARY(popcnt)
3689 [ # # # # : 0 : UNARY(operator not)
# # # # ]
3690 : : #undef UNARY
3691 : :
3692 : : /** Concatenates the most significant bit (or the boolean value if `this` is boolean) of each value of `this`
3693 : : * into a single mask. */
3694 : 0 : PrimitiveExpr<uint32_t, 1> bitmask() requires (L <= 32) and requires (vector_type v) { v.bitmask(); } {
3695 [ # # ]: 0 : std::optional<PrimitiveExpr<uint32_t, 1>> res = vectors_[0].bitmask();
3696 [ # # ]: 0 : for (std::size_t idx = 1; idx < num_vectors; ++idx)
3697 [ # # # # : 0 : res.emplace((vectors_[idx].bitmask() << uint32_t(idx * vector_type::num_simd_lanes)) bitor *res);
# # # # #
# ]
3698 [ # # ]: 0 : return *res;
3699 : 0 : }
3700 : : /** Concatenates the most significant bit (or the boolean value if `this` is boolean) of each value of `this`
3701 : : * into a single mask. */
3702 : : PrimitiveExpr<uint64_t, 1> bitmask() requires (L > 32) and (L <= 64) and requires (vector_type v) { v.bitmask(); } {
3703 : : std::optional<PrimitiveExpr<uint64_t, 1>> res = vectors_[0].bitmask();
3704 : : for (std::size_t idx = 1; idx < num_vectors; ++idx)
3705 : : res.emplace((vectors_[idx].bitmask() << uint64_t(idx * vector_type::num_simd_lanes)) bitor *res);
3706 : : return *res;
3707 : : }
3708 : :
3709 : : /** Returns `true` iff any value is `true` or rather non-zero. */
3710 : : PrimitiveExpr<bool, 1> any_true() requires requires (vector_type v) { v.any_true(); } {
3711 : : std::optional<PrimitiveExpr<bool, 1>> res = vectors_[0].any_true();
3712 : : for (std::size_t idx = 1; idx < num_vectors; ++idx)
3713 : : res.emplace(vectors_[idx].any_true() or *res);
3714 : : return *res;
3715 : : }
3716 : :
3717 : : /** Returns `true` iff all values are `true` or rather non-zero. */
3718 : 0 : PrimitiveExpr<bool, 1> all_true() requires requires (vector_type v) { v.all_true(); } {
3719 [ # # ]: 0 : std::optional<PrimitiveExpr<bool, 1>> res = vectors_[0].all_true();
3720 [ # # ]: 0 : for (std::size_t idx = 1; idx < num_vectors; ++idx)
3721 [ # # # # : 0 : res.emplace(vectors_[idx].all_true() and *res);
# # # # ]
3722 [ # # ]: 0 : return *res;
3723 : 0 : }
3724 : :
3725 : :
3726 : : /*------------------------------------------------------------------------------------------------------------------
3727 : : * Binary operations
3728 : : *----------------------------------------------------------------------------------------------------------------*/
3729 : :
3730 : : #define BINARY(OP) \
3731 : : template<dsl_primitive U> \
3732 : : requires arithmetically_combinable<T, U, L> and \
3733 : : requires (typename PrimitiveExpr<common_type_t<T, U>, L>::vector_type left, \
3734 : : typename PrimitiveExpr<common_type_t<T, U>, L>::vector_type right) \
3735 : : { left.OP(right); } \
3736 : : auto OP(PrimitiveExpr<U, L> other) { \
3737 : : using To = common_type_t<T, U>; \
3738 : : using OpT = decltype(to<To, L>()); \
3739 : : using ResVecT = \
3740 : : decltype(std::declval<typename OpT::vector_type>().OP(std::declval<typename OpT::vector_type>())); \
3741 : : static_assert(ResVecT::num_simd_lanes * sizeof(typename ResVecT::type) == 16, \
3742 : : "result vectors must be fully utilized"); \
3743 : : auto this_converted = this->template to<To, L>(); \
3744 : : auto other_converted = other.template to<To, L>(); \
3745 : : std::array<ResVecT, OpT::num_vectors> vectors; \
3746 : : for (std::size_t idx = 0; idx < OpT::num_vectors; ++idx) \
3747 : : vectors[idx] = this_converted.vectors_[idx].OP(other_converted.vectors_[idx]); \
3748 : : return PrimitiveExpr<typename ResVecT::type, ResVecT::num_simd_lanes * OpT::num_vectors>(std::move(vectors)); \
3749 : : }
3750 : :
3751 [ # # # # : 0 : BINARY(operator +)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3752 [ # # # # : 0 : BINARY(operator -)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3753 [ # # # # : 0 : BINARY(operator *)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3754 [ # # # # : 0 : BINARY(operator /)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3755 [ # # # # : 0 : BINARY(min)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3756 [ # # # # : 0 : BINARY(max)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3757 : : BINARY(avg)
3758 [ # # # # : 0 : BINARY(operator bitand)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3759 : : BINARY(operator bitor)
3760 [ # # # # : 0 : BINARY(operator xor)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3761 [ # # # # : 0 : BINARY(operator and)
# # # # #
# # # ]
3762 : : BINARY(and_not)
3763 [ # # # # : 0 : BINARY(operator or)
# # # # #
# # # ]
3764 : : #undef BINARY
3765 : :
3766 : : #define SHIFT(OP) \
3767 : : template<dsl_primitive U> \
3768 : : PrimitiveExpr OP(PrimitiveExpr<U, 1> other) requires requires (vector_type v) { v.OP(other); } { \
3769 : : std::array<vector_type, num_vectors> vectors; \
3770 : : for (std::size_t idx = 0; idx < num_vectors; ++idx) \
3771 : : vectors[idx] = vectors_[idx].OP(other.clone()); \
3772 : : other.discard(); \
3773 : : return PrimitiveExpr(std::move(vectors)); \
3774 : : }
3775 : :
3776 : : SHIFT(operator <<)
3777 : : SHIFT(operator >>)
3778 : : #undef SHIFT
3779 : :
3780 : : #define BINVOP_(NAME, SIGN, TYPE) (::wasm::BinaryOp::NAME##SIGN##Vec##TYPE)
3781 : : #define BINIVOP_(NAME, SIGN) [] { \
3782 : : if constexpr (sizeof(To) == 8) \
3783 : : return BINVOP_(NAME,SIGN,I64x2); \
3784 : : else if constexpr (sizeof(To) == 4) \
3785 : : return BINVOP_(NAME,SIGN,I32x4); \
3786 : : else if constexpr (sizeof(To) == 2) \
3787 : : return BINVOP_(NAME,SIGN,I16x8); \
3788 : : else if constexpr (sizeof(To) == 1) \
3789 : : return BINVOP_(NAME,SIGN,I8x16); \
3790 : : else \
3791 : : M_unreachable("unsupported operation"); \
3792 : : } ()
3793 : : #define BINFVOP_(NAME) [] { \
3794 : : if constexpr (sizeof(To) == 8) \
3795 : : return BINVOP_(NAME,,F64x2); \
3796 : : else if constexpr (sizeof(To) == 4) \
3797 : : return BINVOP_(NAME,,F32x4); \
3798 : : else \
3799 : : M_unreachable("unsupported operation"); \
3800 : : } ()
3801 : : #define BINARY_VOP(NAME, SIGN) [] { \
3802 : : if constexpr (std::integral<To>) \
3803 : : return BINIVOP_(NAME, SIGN); \
3804 : : else if constexpr (std::floating_point<To>) \
3805 : : return BINFVOP_(NAME); \
3806 : : else \
3807 : : M_unreachable("unsupported operation"); \
3808 : : } ()
3809 : :
3810 : : private:
3811 : : /** Transforms a comparison result into its boolean representation. */
3812 : 12 : PrimitiveExpr<bool, L> cmp_helper() requires unsigned_integral<T> {
3813 : : if constexpr (L > 16) { // enough values present s.t. integer narrowing can be used on multiple vectors
3814 : 0 : auto narrowed = to<uint8_t, L>();
3815 [ # # # # : 0 : return PrimitiveExpr<bool, L>(narrowed.template move<bool, L>());
# # # # #
# # # # #
# # ]
3816 : 0 : } else if constexpr (L == 16) { // enough values present s.t. integer narrowing can be used on single vector
3817 : 12 : auto narrowed = to<uint8_t, L>();
3818 [ # # # # : 12 : return PrimitiveExpr<bool, L>(narrowed.move());
+ - - + #
# # # ]
3819 : 12 : } else if constexpr (num_vectors == 2) { // swizzle bytes of two vectors together
3820 [ # # ]: 0 : PrimitiveExpr<bool, L * sizeof(T)> cmp(move<bool, L * sizeof(T)>());
3821 : : std::array<uint8_t, L> indices;
3822 [ # # ]: 0 : for (std::size_t idx = 0; idx < L; ++idx)
3823 : 0 : indices[idx] = idx * sizeof(T);
3824 [ # # ]: 0 : return cmp.swizzle_bytes(indices);
3825 : 0 : } else { // shuffle bytes of four vectors together
3826 : 0 : auto vectors = move<bool, L * sizeof(T)>();
3827 : : static_assert(vectors.size() == 4);
3828 : : std::array<uint8_t, L / 2> indices;
3829 [ # # ]: 0 : for (std::size_t idx = 0; idx < L / 2; ++idx)
3830 : 0 : indices[idx] = idx * sizeof(T);
3831 [ # # ]: 0 : auto low = ShuffleBytes(vectors[0], vectors[1], indices);
3832 [ # # ]: 0 : auto high = ShuffleBytes(vectors[2], vectors[3], indices);
3833 : : std::array<uint8_t, L> lanes;
3834 : 0 : std::iota(lanes.begin(), lanes.end(), 0); // fill with [0, L), i.e. all L/2 lanes of low and high concatenated
3835 [ # # ]: 0 : return ShuffleLanes(low, high, lanes);
3836 : 0 : }
3837 : 12 : }
3838 : :
3839 : : public:
3840 : : /** Checks whether `this` equals \p other. */
3841 : : template<dsl_primitive U>
3842 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
3843 : 2 : PrimitiveExpr<bool, L> operator==(PrimitiveExpr<U, L> other) {
3844 : : using To = common_type_t<T, U>;
3845 : : using OpT = decltype(to<To, L>());
3846 : : static constexpr std::size_t lanes = OpT::vector_type::num_simd_lanes;
3847 : 2 : auto this_converted = this->template to<To, L>();
3848 [ # # # # : 2 : auto other_converted = other.template to<To, L>();
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3849 : 2 : std::array<PrimitiveExpr<uint_t<sizeof(To)>, lanes>, OpT::num_vectors> vectors;
3850 [ # # # # : 10 : for (std::size_t idx = 0; idx < OpT::num_vectors; ++idx)
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3851 [ # # # # : 16 : vectors[idx] = this_converted.vectors_[idx].template binary<uint_t<sizeof(To)>, lanes>(
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3852 [ # # # # : 24 : BINARY_VOP(Eq,), other_converted.vectors_[idx]
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3853 : : );
3854 [ # # # # : 2 : return PrimitiveExpr<uint_t<sizeof(To)>, L>(std::move(vectors)).cmp_helper();
# # # # #
# # # + -
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3855 : 2 : }
3856 : :
3857 : : /** Checks whether `this` unequal to \p other. */
3858 : : template<dsl_primitive U>
3859 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
3860 : 2 : PrimitiveExpr<bool, L> operator!=(PrimitiveExpr<U, L> other) {
3861 : : using To = common_type_t<T, U>;
3862 : : using OpT = decltype(to<To, L>());
3863 : : static constexpr std::size_t lanes = OpT::vector_type::num_simd_lanes;
3864 : 2 : auto this_converted = this->template to<To, L>();
3865 [ # # # # : 2 : auto other_converted = other.template to<To, L>();
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3866 : 2 : std::array<PrimitiveExpr<uint_t<sizeof(To)>, lanes>, OpT::num_vectors> vectors;
3867 [ # # # # : 10 : for (std::size_t idx = 0; idx < OpT::num_vectors; ++idx)
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3868 [ # # # # : 16 : vectors[idx] = this_converted.vectors_[idx].template binary<uint_t<sizeof(To)>, lanes>(
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3869 [ # # # # : 24 : BINARY_VOP(Ne,), other_converted.vectors_[idx]
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3870 : : );
3871 [ # # # # : 2 : return PrimitiveExpr<uint_t<sizeof(To)>, L>(std::move(vectors)).cmp_helper();
# # # # #
# # # + -
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3872 : 2 : }
3873 : :
3874 : : /** Checks whether `this` less than \p other. */
3875 : : template<arithmetic U>
3876 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
3877 : 2 : PrimitiveExpr<bool, L> operator<(PrimitiveExpr<U, L> other) requires arithmetic<T> {
3878 : : using To = common_type_t<T, U>;
3879 : : using OpT = decltype(to<To, L>());
3880 : : static constexpr std::size_t lanes = OpT::vector_type::num_simd_lanes;
3881 : 4 : auto op = [](){
3882 : : if constexpr (std::integral<To>) {
3883 : : if constexpr (sizeof(To) == 8)
3884 : 0 : return BINVOP_(Lt, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
3885 : : else if constexpr (sizeof(To) == 4)
3886 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Lt, S, I32x4), BINVOP_(Lt, U, I32x4));
3887 : : else if constexpr (sizeof(To) == 2)
3888 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Lt, S, I16x8), BINVOP_(Lt, U, I16x8));
3889 : : else if constexpr (sizeof(To) == 1)
3890 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Lt, S, I8x16), BINVOP_(Lt, U, I8x16));
3891 : : } else if (std::floating_point<To>) {
3892 : 0 : return BINFVOP_(Lt);
3893 : : }
3894 : : M_unreachable("unsupported operation");
3895 : : }();
3896 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
3897 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
3898 : 2 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
3899 : : auto _this =
3900 : 4 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
3901 : : auto _other =
3902 [ # # + - : 4 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3903 [ # # + - : 2 : auto this_converted = _this.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3904 [ # # + - : 2 : auto other_converted = _other.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3905 : 2 : std::array<PrimitiveExpr<uint_t<sizeof(To)>, lanes>, OpT::num_vectors> vectors;
3906 [ # # + + : 10 : for (std::size_t idx = 0; idx < OpT::num_vectors; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3907 [ # # + - : 8 : vectors[idx] = this_converted.vectors_[idx].template binary<uint_t<sizeof(To)>, lanes>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3908 [ # # + - : 8 : op, other_converted.vectors_[idx]
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3909 : : );
3910 [ # # # # : 2 : return PrimitiveExpr<uint_t<sizeof(To)>, L>(std::move(vectors)).cmp_helper();
# # + - +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3911 : 2 : }
3912 : :
3913 : : /** Checks whether `this` less than or equals to \p other. */
3914 : : template<arithmetic U>
3915 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
3916 : 2 : PrimitiveExpr<bool, L> operator<=(PrimitiveExpr<U, L> other) requires arithmetic<T> {
3917 : : using To = common_type_t<T, U>;
3918 : : using OpT = decltype(to<To, L>());
3919 : : static constexpr std::size_t lanes = OpT::vector_type::num_simd_lanes;
3920 : 4 : auto op = [](){
3921 : : if constexpr (std::integral<To>) {
3922 : : if constexpr (sizeof(To) == 8)
3923 : 0 : return BINVOP_(Le, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
3924 : : else if constexpr (sizeof(To) == 4)
3925 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Le, S, I32x4), BINVOP_(Le, U, I32x4));
3926 : : else if constexpr (sizeof(To) == 2)
3927 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Le, S, I16x8), BINVOP_(Le, U, I16x8));
3928 : : else if constexpr (sizeof(To) == 1)
3929 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Le, S, I8x16), BINVOP_(Le, U, I8x16));
3930 : : } else if (std::floating_point<To>) {
3931 : 0 : return BINFVOP_(Le);
3932 : : }
3933 : : M_unreachable("unsupported operation");
3934 : : }();
3935 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
3936 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
3937 : 2 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
3938 : : auto _this =
3939 : 4 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
3940 : : auto _other =
3941 [ # # + - : 4 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3942 [ # # + - : 2 : auto this_converted = _this.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3943 [ # # + - : 2 : auto other_converted = _other.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3944 : 2 : std::array<PrimitiveExpr<uint_t<sizeof(To)>, lanes>, OpT::num_vectors> vectors;
3945 [ # # + + : 10 : for (std::size_t idx = 0; idx < OpT::num_vectors; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3946 [ # # + - : 8 : vectors[idx] = this_converted.vectors_[idx].template binary<uint_t<sizeof(To)>, lanes>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3947 [ # # + - : 8 : op, other_converted.vectors_[idx]
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3948 : : );
3949 [ # # # # : 2 : return PrimitiveExpr<uint_t<sizeof(To)>, L>(std::move(vectors)).cmp_helper();
# # + - +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3950 : 2 : }
3951 : :
3952 : : /** Checks whether `this` greater than to \p other. */
3953 : : template<arithmetic U>
3954 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
3955 : 2 : PrimitiveExpr<bool, L> operator>(PrimitiveExpr<U, L> other) requires arithmetic<T> {
3956 : : using To = common_type_t<T, U>;
3957 : : using OpT = decltype(to<To, L>());
3958 : : static constexpr std::size_t lanes = OpT::vector_type::num_simd_lanes;
3959 : 4 : auto op = [](){
3960 : : if constexpr (std::integral<To>) {
3961 : : if constexpr (sizeof(To) == 8)
3962 : 0 : return BINVOP_(Gt, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
3963 : : else if constexpr (sizeof(To) == 4)
3964 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Gt, S, I32x4), BINVOP_(Gt, U, I32x4));
3965 : : else if constexpr (sizeof(To) == 2)
3966 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Gt, S, I16x8), BINVOP_(Gt, U, I16x8));
3967 : : else if constexpr (sizeof(To) == 1)
3968 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Gt, S, I8x16), BINVOP_(Gt, U, I8x16));
3969 : : } else if (std::floating_point<To>) {
3970 : 0 : return BINFVOP_(Gt);
3971 : : }
3972 : : M_unreachable("unsupported operation");
3973 : : }();
3974 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
3975 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
3976 : 2 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
3977 : : auto _this =
3978 : 4 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
3979 : : auto _other =
3980 [ # # + - : 4 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3981 [ # # + - : 2 : auto this_converted = _this.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3982 [ # # + - : 2 : auto other_converted = _other.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3983 : 2 : std::array<PrimitiveExpr<uint_t<sizeof(To)>, lanes>, OpT::num_vectors> vectors;
3984 [ # # + + : 10 : for (std::size_t idx = 0; idx < OpT::num_vectors; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3985 [ # # + - : 8 : vectors[idx] = this_converted.vectors_[idx].template binary<uint_t<sizeof(To)>, lanes>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3986 [ # # + - : 8 : op, other_converted.vectors_[idx]
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3987 : : );
3988 [ # # # # : 2 : return PrimitiveExpr<uint_t<sizeof(To)>, L>(std::move(vectors)).cmp_helper();
# # + - +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3989 : 2 : }
3990 : :
3991 : : /** Checks whether `this` greater than or equals to \p other. */
3992 : : template<arithmetic U>
3993 : : requires same_signedness<T, U> and arithmetically_combinable<T, U, L>
3994 : 2 : PrimitiveExpr<bool, L> operator>=(PrimitiveExpr<U, L> other) requires arithmetic<T> {
3995 : : using To = common_type_t<T, U>;
3996 : : using OpT = decltype(to<To, L>());
3997 : : static constexpr std::size_t lanes = OpT::vector_type::num_simd_lanes;
3998 : 4 : auto op = [](){
3999 : : if constexpr (std::integral<To>) {
4000 : : if constexpr (sizeof(To) == 8)
4001 : 0 : return BINVOP_(Ge, S, I64x2); // unsigned comparison missing, use signed but flip MSB of operands
4002 : : else if constexpr (sizeof(To) == 4)
4003 : 4 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Ge, S, I32x4), BINVOP_(Ge, U, I32x4));
4004 : : else if constexpr (sizeof(To) == 2)
4005 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Ge, S, I16x8), BINVOP_(Ge, U, I16x8));
4006 : : else if constexpr (sizeof(To) == 1)
4007 : 0 : return M_CONSTEXPR_COND(std::is_signed_v<To>, BINVOP_(Ge, S, I8x16), BINVOP_(Ge, U, I8x16));
4008 : : } else if (std::floating_point<To>) {
4009 : 0 : return BINFVOP_(Ge);
4010 : : }
4011 : : M_unreachable("unsupported operation");
4012 : : }();
4013 : : /* Wasm does not support comparison of U64 SIMD vectors. Thus, flip the MSB in each lane (which
4014 : : * basically shifts the unsigned domain into the signed one) and use the signed comparison afterwards. */
4015 : 2 : constexpr bool is_u64_vec = unsigned_integral<To> and sizeof(To) == 8 and L > 1;
4016 : : auto _this =
4017 : 4 : M_CONSTEXPR_COND(is_u64_vec, (*this xor PrimitiveExpr<T, L>(T(1) << (CHAR_BIT * sizeof(T) - 1))), *this);
4018 : : auto _other =
4019 [ # # + - : 4 : M_CONSTEXPR_COND(is_u64_vec, (other xor PrimitiveExpr<U, L>(U(1) << (CHAR_BIT * sizeof(U) - 1))), other);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4020 [ # # + - : 2 : auto this_converted = _this.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4021 [ # # + - : 2 : auto other_converted = _other.template to<To, L>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4022 : 2 : std::array<PrimitiveExpr<uint_t<sizeof(To)>, lanes>, OpT::num_vectors> vectors;
4023 [ # # + + : 10 : for (std::size_t idx = 0; idx < OpT::num_vectors; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4024 [ # # + - : 8 : vectors[idx] = this_converted.vectors_[idx].template binary<uint_t<sizeof(To)>, lanes>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4025 [ # # + - : 8 : op, other_converted.vectors_[idx]
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4026 : : );
4027 [ # # # # : 2 : return PrimitiveExpr<uint_t<sizeof(To)>, L>(std::move(vectors)).cmp_helper();
# # + - +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4028 : 2 : }
4029 : :
4030 : : #undef BINARY_VOP
4031 : : #undef BINFVOP_
4032 : : #undef BINIVOP_
4033 : : #undef BINVOP_
4034 : :
4035 : :
4036 : : /*------------------------------------------------------------------------------------------------------------------
4037 : : * Modifications
4038 : : *----------------------------------------------------------------------------------------------------------------*/
4039 : :
4040 : : /** Extracts the \tparam M -th value of `this`. */
4041 : : template<std::size_t M>
4042 : : requires (M < L)
4043 : 0 : PrimitiveExpr<T, 1> extract() {
4044 [ # # # # : 0 : auto res = vectors_[M / vector_type::num_simd_lanes].clone().template extract<M % vector_type::num_simd_lanes>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4045 [ # # # # : 0 : discard(); // to discard all vectors not used for extraction
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4046 : 0 : return res;
4047 [ # # # # : 0 : }
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4048 : :
4049 : : /** Replaces the \tparam M -th value of `this` with \p value. */
4050 : : template<std::size_t M, primitive_convertible U>
4051 : : requires (M < L)
4052 : : PrimitiveExpr replace(U &&value) requires requires (vector_type v) { v.replace<0>(std::forward<U>(value)); } {
4053 : : static constexpr std::size_t lanes = vector_type::num_simd_lanes;
4054 : : vectors_[M / lanes] = vectors_[M / lanes].template replace<M % lanes>(std::forward<U>(value));
4055 : : return *this;
4056 : : }
4057 : :
4058 : : /** Selects lanes of `this` in byte granularity depending on the indices specified by \p indices. Indices `i` in
4059 : : * the range [0, L * sizeof(T)) select the `i`-th` lane, indices outside of this range result in undefined values. */
4060 : : template<std::size_t M>
4061 : : requires (M > 0) and (M <= 16) and (M % sizeof(T) == 0)
4062 : 0 : PrimitiveExpr<T, M / sizeof(T)> swizzle_bytes(const std::array<uint8_t, M> &indices) requires (num_vectors == 2) {
4063 [ # # # # ]: 0 : return Module::Get().emit_shuffle_bytes(vectors_[0], vectors_[1], indices);
4064 : 0 : }
4065 : :
4066 : : /** Selects lanes of `this` in lane granularity depending on the indices specified by \p indices. Indices `i` in
4067 : : * the range [0, L) select the `i`-th` lane, indices outside of this range result in undefined values. */
4068 : : template<std::size_t M>
4069 : : requires (M > 0) and (is_pow_2(M)) and (M * sizeof(T) <= 16)
4070 : 4 : PrimitiveExpr<T, M> swizzle_lanes(const std::array<uint8_t, M> &indices) requires (num_vectors == 2) {
4071 [ + - - + ]: 4 : return Module::Get().emit_shuffle_lanes(vectors_[0], vectors_[1], indices);
4072 : 0 : }
4073 : :
4074 : :
4075 : : /*------------------------------------------------------------------------------------------------------------------
4076 : : * Printing
4077 : : *----------------------------------------------------------------------------------------------------------------*/
4078 : :
4079 : : friend std::ostream & operator<<(std::ostream &out, const PrimitiveExpr &P) {
4080 : : out << "PrimitiveExpr<" << typeid(T).name() << "," << L << ">: [";
4081 : : for (auto it = P.vectors_.cbegin(); it != P.vectors_.cend(); ++it) {
4082 : : if (it != P.vectors_.cbegin())
4083 : : out << ", ";
4084 : : out << *it;
4085 : : }
4086 : : out << "]";
4087 : : return out;
4088 : : }
4089 : :
4090 : : void dump(std::ostream &out) const { out << *this << std::endl; }
4091 : : void dump() const { dump(std::cerr); }
4092 : : };
4093 : :
4094 : :
4095 : : /*======================================================================================================================
4096 : : * Define binary operators on `PrimitiveExpr`
4097 : : *====================================================================================================================*/
4098 : :
4099 : : /** List of supported binary operators on `PrimitiveExpr`, `Expr`, `Variable`, etc. */
4100 : : #define BINARY_LIST(X) \
4101 : : X(operator +) \
4102 : : X(operator -) \
4103 : : X(operator *) \
4104 : : X(operator /) \
4105 : : X(operator %) \
4106 : : X(operator bitand) \
4107 : : X(operator bitor) \
4108 : : X(operator xor) \
4109 : : X(operator <<) \
4110 : : X(operator >>) \
4111 : : X(operator ==) \
4112 : : X(operator !=) \
4113 : : X(operator <) \
4114 : : X(operator <=) \
4115 : : X(operator >) \
4116 : : X(operator >=) \
4117 : : X(operator and) \
4118 : : X(operator or) \
4119 : : X(copy_sign) \
4120 : : X(min) \
4121 : : X(max) \
4122 : : X(avg) \
4123 : : X(rotl) \
4124 : : X(rotr) \
4125 : : X(and_not)
4126 : :
4127 : : /*----- Forward binary operators on operands convertible to PrimitiveExpr<T, L> --------------------------------------*/
4128 : : #define MAKE_BINARY(OP) \
4129 : : template<primitive_convertible T, primitive_convertible U> \
4130 : : requires requires (primitive_expr_t<T> t, primitive_expr_t<U> u) { t.OP(u); } \
4131 : : auto OP(T &&t, U &&u) \
4132 : : { \
4133 : : return primitive_expr_t<T>(std::forward<T>(t)).OP(primitive_expr_t<U>(std::forward<U>(u))); \
4134 : : }
4135 [ + - - + : 7530 : BINARY_LIST(MAKE_BINARY)
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ # # # #
+ - - + +
- - + # #
# # # # #
# # # # #
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + # # #
# # # # #
+ - - + +
- - + + -
- + + - -
+ + - - +
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
+ - - + +
- - + # #
# # + - -
+ + - - +
+ - - + +
- - + # #
# # + - -
+ # # # #
# # # # +
- - + # #
# # + - -
+ + - - +
# # # # +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + # # #
# # # # #
# # # # #
# # # # #
# # + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # +
- - + + -
- + + - -
+ + - - +
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
+ + - - +
+ - - + +
- - + + -
- + # # #
# # # # #
# # # # #
# # # + -
- + + - -
+ + - - +
# # # # +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + #
# # # + -
- + + - -
+ + - - +
+ - - + #
# # # # #
# # + - -
+ ]
4136 : : #undef MAKE_BINARY
4137 : :
4138 : : /** Specialization of `PrimitiveExpr<T, L>` for pointer to primitive type \tparam T. Represents an expression (AST)
4139 : : * evaluating to \tparam L runtime values of pointer to primitive type \tparam T. */
4140 : : template<dsl_pointer_to_primitive T, std::size_t L>
4141 : : requires (L > 0) and (is_pow_2(L)) and
4142 : : ((L == 1) or requires { PrimitiveExpr<std::remove_pointer_t<T>, L>(); })
4143 : : struct PrimitiveExpr<T, L>
4144 : : {
4145 : : using type = T;
4146 : : static constexpr std::size_t num_simd_lanes = L;
4147 : : using pointed_type = std::decay_t<std::remove_pointer_t<T>>;
4148 : : using offset_t = int32_t;
4149 : :
4150 : : /*----- Friends --------------------------------------------------------------------------------------------------*/
4151 : : template<typename, std::size_t> friend struct PrimitiveExpr; // to convert U* to T* and to convert uint32_t to T*
4152 : : template<typename, VariableKind, bool, std::size_t>
4153 : : friend class detail::variable_storage; // to construct from `::wasm::Expression` and access private `expr()`
4154 : : friend struct Module; // to acces internal ::wasm::Expr
4155 : : template<typename> friend struct FunctionProxy; // to access internal `::wasm::Expr` to construct function calls
4156 : : template<dsl_primitive, std::size_t, bool> friend struct detail::the_reference; // to access load()/store()
4157 : : template<typename> friend struct invoke_interpreter; // to access private `expr()`
4158 : :
4159 : : private:
4160 : : PrimitiveExpr<uint32_t, 1> addr_; ///< the address into the Wasm linear memory
4161 : : offset_t offset_ = 0; ///< offset to this in bytes; used to directly address pointer via base address and offset
4162 : :
4163 : : public:
4164 : : /** Constructs a `PrimitiveExpr` from the memory address \p addr. Optionally accepts an \p offset. */
4165 : 5604 : explicit PrimitiveExpr(PrimitiveExpr<uint32_t, 1> addr, offset_t offset = 0) : addr_(addr), offset_(offset) { }
4166 : :
4167 : : private:
4168 : : /** Constructs a `PrimitiveExpr` from the given address \p addr and \p referenced_bits. Optionally accepts an \p
4169 : : * offset. */
4170 : 448 : explicit PrimitiveExpr(::wasm::Expression *addr, std::list<std::shared_ptr<Bit>> referenced_bits = {},
4171 : : offset_t offset = 0)
4172 [ + - + - : 448 : : addr_(addr, std::move(referenced_bits))
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # # #
# # + - #
# # # # #
# # # # #
# # # + -
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4173 : 448 : , offset_(offset)
4174 : 448 : { }
4175 : : /** Constructs a `PrimitiveExpr` from a `std::pair` \p addr of the addres and the shared bits. Optionally
4176 : : * accepts an \p offset. */
4177 : 418 : explicit PrimitiveExpr(std::pair<::wasm::Expression*, std::list<std::shared_ptr<Bit>>> addr, offset_t offset = 0)
4178 [ + - + - : 418 : : PrimitiveExpr(std::move(addr.first), std::move(addr.second), offset)
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # +
- + - # #
# # # # #
# + - # #
# # # # #
# # # # #
# # + - #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4179 : 418 : { }
4180 : :
4181 : : public:
4182 : 0 : PrimitiveExpr(T raw_ptr) requires (L == 1)
4183 : 0 : : addr_(0U)
4184 [ # # # # : 0 : , offset_([&raw_ptr](){
# # # # #
# # # # #
# # ]
4185 : 0 : auto &memory = Module::Memory();
4186 : 0 : const auto offset = reinterpret_cast<uint8_t*>(raw_ptr) - static_cast<uint8_t*>(memory.addr());
4187 [ # # # # : 0 : M_insist(offset >= 0 and offset < memory.size(), "invalid raw pointer");
# # # # #
# # # # #
# # ]
4188 : 0 : return offset;
4189 : : }())
4190 : 0 : { }
4191 : : PrimitiveExpr(const PrimitiveExpr&) = delete;
4192 : : /** Constructs a new `PrimitiveExpr` by **moving** the underlying `expr_`, `referenced_bits`, and `offset_`
4193 : : * of `other` to `this`. */
4194 : 3612 : PrimitiveExpr(PrimitiveExpr &other) : addr_(other.addr_), offset_(other.offset_) { /* move, not copy */ }
4195 : : /** Constructs a new `PrimitiveExpr` by **moving** the underlying `expr_`, `referenced_bits`, and `offset_`
4196 : : * of `other` to `this`. */
4197 : 1056 : PrimitiveExpr(PrimitiveExpr &&other) : addr_(other.addr_), offset_(other.offset_) { }
4198 : :
4199 : : PrimitiveExpr & operator=(PrimitiveExpr&&) = delete;
4200 : :
4201 : : /** Constructs a Wasm `nullptr`. Note, that in order to implement `nullptr` in Wasm, we must create an artificial
4202 : : * address that cannot be accessed. */
4203 [ + - # # ]: 10 : static PrimitiveExpr Nullptr() { return PrimitiveExpr(PrimitiveExpr<uint32_t, 1>(0U)); }
4204 : :
4205 : : private:
4206 : : /** **Moves** the underlying Binaryen `::wasm::Expression` out of `this`. */
4207 [ + - + - : 280 : ::wasm::Expression * expr() { return to<uint32_t>().expr(); }
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# ]
4208 : : /** **Moves** the referenced bits out of `this`. */
4209 : 20 : std::list<std::shared_ptr<Bit>> referenced_bits() { return addr_.referenced_bits(); }
4210 : : /** **Moves** the underlying Binaryen `wasm::Expression` and the referenced bits out of `this`. */
4211 : : std::pair<::wasm::Expression*, std::list<std::shared_ptr<Bit>>> move() { return addr_.move(); }
4212 : :
4213 : : public:
4214 : : /** Returns `true` if this `PrimitiveExpr` actually holds a value (Binaryen AST), `false` otherwise. Can be used to
4215 : : * test whether this `PrimitiveExpr` has already been used. */
4216 : 1254 : explicit operator bool() const { return bool(addr_); }
4217 : :
4218 : : /** Creates and returns a *deep copy* of `this`. */
4219 [ + - # # : 2284 : PrimitiveExpr clone() const { return PrimitiveExpr(addr_.clone(), offset_); }
+ - + - +
- # # + -
+ - + - +
- + - # #
+ - # # #
# + - # #
# # # # +
- # # # #
# # # # #
# # # # #
# # + - #
# # # # #
# # # # #
# + - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
# # # # #
# # # #
# ]
4220 : :
4221 : : /** Discards `this`. This is necessary to signal in our DSL that a value is *expectedly* unused (and not dead
4222 : : * code). For example, the return value of a function that was invoked because of its side effects may remain
4223 : : * unused. One **must** discard the returned value to signal that the value is expectedly left unused. */
4224 : 66 : void discard() { addr_.discard(); }
4225 : :
4226 : :
4227 : : /*------------------------------------------------------------------------------------------------------------------
4228 : : * Conversion operations
4229 : : *----------------------------------------------------------------------------------------------------------------*/
4230 : :
4231 : : public:
4232 : : /** Explicit conversion of a `PrimitiveExpr<void*, 1>` to a `PrimitiveExpr<To, ToL>`. Only applicable if \tparam To
4233 : : * is a pointer to primitive type. */
4234 : : template<dsl_pointer_to_primitive To, std::size_t ToL = L>
4235 : : requires (not std::is_void_v<std::remove_pointer_t<To>>)
4236 : 416 : PrimitiveExpr<To, ToL> to() requires std::is_void_v<pointed_type> and (L == 1) {
4237 [ + - + - : 416 : Wasm_insist((clone().template to<uint32_t>() % uint32_t(alignof(std::remove_pointer_t<To>))).eqz(),
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # #
# # # # #
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4238 : : "cannot convert to type whose alignment requirement is not fulfilled");
4239 [ - + - + : 416 : return PrimitiveExpr<To, ToL>(addr_.move(), offset_);
- + - + -
+ - + - +
- + - + -
+ - + - +
# # - + -
+ # # # #
# # # # -
+ # # # #
# # # # #
# # # # #
- + # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4240 : 0 : }
4241 : :
4242 : : /** Explicit conversion of a `PrimitiveExpr<T*, L>` to a `PrimitiveExpr<uint32_t, 1>`. Adds possible offset to
4243 : : * the pointer. */
4244 : : template<typename To, std::size_t ToL = 1>
4245 : : requires std::same_as<To, uint32_t> and (ToL == 1)
4246 : 3452 : PrimitiveExpr<uint32_t, 1> to() {
4247 [ + + - + : 3452 : return offset_ ? (offset_ > 0 ? addr_ + uint32_t(offset_) : addr_ - uint32_t(-offset_)) : addr_;
+ + - + +
+ - + - +
# # + + -
+ # # # #
+ + + - +
+ - + + +
- + - + #
# - + # #
- + # # -
+ # # # #
# # - + #
# - + # #
# # # # #
# # # # #
# # # # #
# - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4248 : : }
4249 : :
4250 : : /** Explicit conversion of a `PrimitiveExpr<T*, L>` to a `PrimitiveExpr<void*, 1>`. */
4251 : : template<typename To, std::size_t ToL = 1>
4252 : : requires (not std::same_as<To, T>) and std::same_as<To, void*> and (ToL == 1)
4253 [ + - ]: 2 : PrimitiveExpr<void*, 1> to() { return PrimitiveExpr<void*, 1>(addr_.move(), offset_); }
4254 : :
4255 : : /** Explicit dummy conversion of a `PrimitiveExpr<T*, L>` to a `PrimitiveExpr<T*, L>`. Only needed for convenience
4256 : : * reasons, i.e. to match behaviour of `PrimitiveExpr<dsl_primitive>`. */
4257 : : template<typename To, std::size_t ToL = L>
4258 : : requires std::same_as<To, T> and (L == ToL)
4259 : 20 : PrimitiveExpr to() { return *this; }
4260 : :
4261 : :
4262 : : /*------------------------------------------------------------------------------------------------------------------
4263 : : * Hashing operations
4264 : : *----------------------------------------------------------------------------------------------------------------*/
4265 : :
4266 : : PrimitiveExpr<uint64_t, L> hash() { return to<uint32_t>().hash(); }
4267 : :
4268 : :
4269 : : /*------------------------------------------------------------------------------------------------------------------
4270 : : * Pointer operations
4271 : : *----------------------------------------------------------------------------------------------------------------*/
4272 : :
4273 : : public:
4274 : : /** Returns `true` if `this` is `nullptr`. */
4275 [ + - # # : 1262 : PrimitiveExpr<bool, 1> is_nullptr() { return to<uint32_t>() == 0U; }
+ - + - +
- # # + -
+ - + - +
- + - + -
# # + - +
- # # # #
# # # # +
- # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4276 : :
4277 : : /** Returns `true` if `this` is `NULL`, `false` otherwise. Even if this method performs the same operation
4278 : : * as `is_nullptr()` it should be used for ternary logic since it additionally checks whether ternary logic usage
4279 : : * is expected. */
4280 : 10 : PrimitiveExpr<bool, 1> is_null() {
4281 : : M_insist_no_ternary_logic();
4282 [ + - ]: 10 : return to<uint32_t>() == 0U;
4283 : 0 : }
4284 : :
4285 : : /** Returns `true` if `this` is `NOT NULL`, `false` otherwise. Even if this method performs the same operation
4286 : : * as `not is_nullptr()` it should be used for ternary logic since it additionally checks whether ternary logic
4287 : : * usage is expected. */
4288 : 108 : PrimitiveExpr<bool, 1> not_null() {
4289 : : M_insist_no_ternary_logic();
4290 [ + - ]: 108 : return to<uint32_t>() != 0U;
4291 : 0 : }
4292 : :
4293 : : /** Returns a `std::pair` of `this` and a `PrimitiveExpr<bool, 1>` that tells whether `this` is `nullptr`. */
4294 [ # # # # ]: 0 : std::pair<PrimitiveExpr, PrimitiveExpr<bool, 1>> split() { auto cpy = clone(); return { cpy, is_nullptr() }; }
4295 : :
4296 : : /** Dereferencing a pointer `PrimitiveExpr<T*, L>` yields a `Reference<T, L>`. */
4297 : 1246 : auto operator*() requires dsl_primitive<pointed_type> {
4298 [ + - + - : 1246 : Wasm_insist(not clone().is_nullptr(), "cannot dereference `nullptr`");
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
# # # # +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4299 [ - + - + : 1246 : return Reference<pointed_type, L>(*this);
- + - + #
# - + - +
- + - + -
+ - + - +
# # - + -
+ # # # #
# # # # -
+ # # # #
# # # # #
# # # # #
- + # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4300 : 0 : }
4301 : :
4302 : : /** Dereferencing a `const` pointer `PrimitiveExpr<T*, L>` yields a `ConstReference<T, L>`. */
4303 : : auto operator*() const requires dsl_primitive<pointed_type> {
4304 : : Wasm_insist(not clone().is_nullptr(), "cannot dereference `nullptr`");
4305 : : return ConstReference<pointed_type, L>(*this);
4306 : : }
4307 : :
4308 : : /** Dereferencing and loading a `const` pointer `PrimitiveExpr<T*, L>` yields a `PrimitiveExpr<T, L>`. */
4309 : : PrimitiveExpr<pointed_type, L> operator->() const requires dsl_primitive<pointed_type> {
4310 : : return operator*(); // implicitly convert from ConstReference<pointed_type, L>
4311 : : }
4312 : :
4313 : :
4314 : : /*------------------------------------------------------------------------------------------------------------------
4315 : : * Pointer arithmetic
4316 : : *----------------------------------------------------------------------------------------------------------------*/
4317 : :
4318 : : ///> Adds a \p delta, in elements, to `this`.
4319 : 854 : PrimitiveExpr operator+(PrimitiveExpr<offset_t, 1> delta) {
4320 : : if constexpr (std::is_void_v<pointed_type>) {
4321 [ + - - + ]: 198 : return PrimitiveExpr(addr_ + delta.make_unsigned(), offset_);
4322 : : } else {
4323 : 656 : const uint32_t log_size = std::countr_zero(sizeof(pointed_type));
4324 [ + - + - : 656 : return PrimitiveExpr(addr_ + (delta.make_unsigned() << log_size), offset_);
- + # # #
# # # + -
+ - - + ]
4325 : : }
4326 : 0 : }
4327 : :
4328 : : ///> Adds a \p delta, in elements, to `this`.
4329 : 336 : PrimitiveExpr operator+(offset_t delta) {
4330 : : if constexpr (std::is_void_v<pointed_type>) {
4331 : 54 : offset_ += delta; // in bytes
4332 : : } else {
4333 : 282 : const uint32_t log_size = std::countr_zero(sizeof(pointed_type));
4334 : 282 : offset_ += delta << log_size; // in elements
4335 : : }
4336 : 336 : return *this;
4337 : : }
4338 : :
4339 : : ///> Subtracts a \p delta, in elements, from `this`.
4340 : 0 : PrimitiveExpr operator-(PrimitiveExpr<offset_t, 1> delta) {
4341 : : if constexpr (std::is_void_v<pointed_type>) {
4342 [ # # # # ]: 0 : return PrimitiveExpr(addr_ - delta.make_unsigned(), offset_);
4343 : : } else {
4344 : 0 : const uint32_t log_size = std::countr_zero(sizeof(pointed_type));
4345 [ # # # # : 0 : return PrimitiveExpr(addr_ - (delta.make_unsigned() << log_size), offset_);
# # # # #
# # # ]
4346 : : }
4347 : 0 : }
4348 : :
4349 : : ///> Subtracts a \p delta, in elements, from `this`.
4350 : 4 : PrimitiveExpr operator-(offset_t delta) {
4351 : : if constexpr (std::is_void_v<pointed_type>) {
4352 : 0 : offset_ -= delta; // in bytes
4353 : : } else {
4354 : 4 : const uint32_t log_size = std::countr_zero(sizeof(pointed_type));
4355 : 4 : offset_ -= delta << log_size; // in elements
4356 : : }
4357 : 4 : return *this;
4358 : : }
4359 : :
4360 : : ///> Computes the difference, in elements, between `this` and \p other.
4361 : 4 : PrimitiveExpr<offset_t, 1> operator-(PrimitiveExpr other) {
4362 : : if constexpr (std::is_void_v<pointed_type>) {
4363 [ # # # # ]: 0 : PrimitiveExpr<offset_t, 1> delta_addr = (this->addr_ - other.addr_).make_signed();
4364 : 0 : offset_t delta_offset = this->offset_ - other.offset_;
4365 [ # # # # : 0 : return (delta_offset ? (delta_addr + delta_offset) : delta_addr);
# # ]
4366 : 0 : } else {
4367 : 4 : const int32_t log_size = std::countr_zero(sizeof(pointed_type));
4368 [ + - + - : 4 : PrimitiveExpr<offset_t, 1> delta_addr = (this->addr_ - other.addr_).make_signed() >> log_size;
+ - ]
4369 : 4 : offset_t delta_offset = (this->offset_ - other.offset_) >> log_size;
4370 [ + - # # : 4 : return (delta_offset ? (delta_addr + delta_offset) : delta_addr);
+ - ]
4371 : :
4372 : 4 : }
4373 : 4 : }
4374 : :
4375 : : #define CMP_OP(SYMBOL) \
4376 : : /** Compares `this` to \p other by their addresses. */ \
4377 : : PrimitiveExpr<bool, 1> operator SYMBOL(PrimitiveExpr other) { \
4378 : : return this->to<uint32_t>() SYMBOL other.to<uint32_t>(); \
4379 : : }
4380 [ + - - + : 32 : CMP_OP(==)
# # # # ]
4381 [ + - - + : 120 : CMP_OP(!=)
# # # # ]
4382 [ + - - + : 6 : CMP_OP(<)
# # # # ]
4383 [ + - - + ]: 6 : CMP_OP(<=)
4384 [ + - - + ]: 8 : CMP_OP(>)
4385 [ + - - + ]: 6 : CMP_OP(>=)
4386 : : #undef CMP_OP
4387 : :
4388 : :
4389 : : /*------------------------------------------------------------------------------------------------------------------
4390 : : * Load/Store operations
4391 : : *----------------------------------------------------------------------------------------------------------------*/
4392 : :
4393 : : private:
4394 : 578 : PrimitiveExpr<pointed_type, L> load() requires dsl_primitive<pointed_type> {
4395 : 578 : M_insist(bool(addr_), "address already moved or discarded");
4396 : : if constexpr (L * sizeof(pointed_type) <= 16) {
4397 [ + - + - : 1156 : auto value = Module::Builder().makeLoad(
# # + - +
- # # # #
# # # # +
- + - # #
+ - + - #
# + - + -
# # + - +
- # # # #
# # # # +
- + - # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4398 : 1156 : /* bytes= */ M_CONSTEXPR_COND(L == 1, sizeof(pointed_type), 16),
4399 : : /* signed= */ std::is_signed_v<pointed_type>,
4400 [ + - + - : 578 : /* offset= */ offset_ >= 0 ? offset_ : 0,
# # + - +
- + - + -
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # + -
# # # # #
# # # #
# ]
4401 : : /* align= */ alignof(pointed_type),
4402 [ - + # # : 578 : /* ptr= */ offset_ >= 0 ? addr_.expr() : (addr_ - uint32_t(-offset_)).expr(),
- + # # #
# # # - +
# # - + #
# - + # #
- + # # #
# # # - +
# # - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4403 [ + - + - : 578 : /* type= */ wasm_type<pointed_type, L>(),
# # + - +
- + - + -
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # + -
# # # # #
# # # #
# ]
4404 [ + - + - : 578 : /* memory= */ Module::Get().memory_->name
# # + - +
- + - + -
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # + -
# # # # #
# # # #
# ]
4405 : : );
4406 [ - + - + : 578 : return PrimitiveExpr<pointed_type, L>(value, addr_.referenced_bits());
# # - + -
+ - + - +
# # - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # - +
# # # # #
# # # #
# ]
4407 : : } else {
4408 : : using ResT = PrimitiveExpr<pointed_type, L>;
4409 : 0 : std::array<typename ResT::vector_type, ResT::num_vectors> vectors;
4410 [ # # # # : 0 : for (std::size_t idx = 0; idx < ResT::num_vectors; ++idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4411 [ # # # # : 0 : auto addr_cpy = addr_.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4412 : 0 : auto offset = offset_ + offset_t(idx * 16);
4413 [ # # # # : 0 : auto value = Module::Builder().makeLoad(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4414 : : /* bytes= */ 16,
4415 : : /* signed= */ std::is_signed_v<pointed_type>,
4416 [ # # # # : 0 : /* offset= */ offset >= 0 ? offset : 0,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4417 : : /* align= */ alignof(pointed_type),
4418 [ # # # # : 0 : /* ptr= */ offset >= 0 ? addr_cpy.expr() : (addr_cpy - uint32_t(-offset)).expr(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4419 [ # # # # : 0 : /* type= */ ::wasm::Type(::wasm::Type::v128),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4420 [ # # # # : 0 : /* memory= */ Module::Get().memory_->name
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4421 : : );
4422 [ # # # # : 0 : vectors[idx] = typename ResT::vector_type(value, addr_cpy.referenced_bits());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4423 : 0 : }
4424 [ # # # # : 0 : addr_.discard(); // since it was always cloned
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4425 [ # # # # : 0 : return ResT(std::move(vectors));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4426 : 0 : }
4427 : 0 : }
4428 : :
4429 : 676 : void store(PrimitiveExpr<pointed_type, L> value) requires dsl_primitive<pointed_type> {
4430 : 676 : M_insist(bool(addr_), "address already moved or discarded");
4431 : 676 : M_insist(bool(value), "value already moved or discarded");
4432 : : if constexpr (L * sizeof(pointed_type) <= 16) {
4433 [ # # # # : 1352 : auto e = Module::Builder().makeStore(
# # + - +
- # # + -
+ - # # +
- + - # #
# # # # #
# # # # #
# # + - +
- # # + -
+ - # # +
- + - # #
+ - + - #
# + - + -
# # + - +
- # # # #
# # # # +
- + - # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - # # #
# # # # #
+ - + - #
# # # # #
# # # # #
# # # ]
4434 : 1352 : /* bytes= */ M_CONSTEXPR_COND(L == 1, sizeof(pointed_type), 16),
4435 [ # # + - : 676 : /* offset= */ offset_ >= 0 ? offset_ : 0,
+ - + - #
# # # + -
+ - + - +
- + - + -
# # + - +
- # # # #
# # + - #
# # # # #
+ - # # +
- # # #
# ]
4436 : : /* align= */ alignof(pointed_type),
4437 [ # # # # : 676 : /* ptr= */ offset_ >= 0 ? addr_.expr() : (addr_ - uint32_t(-offset_)).expr(),
- + # # -
+ # # - +
# # # # #
# # # # #
- + # # -
+ # # - +
# # - + #
# - + # #
- + # # #
# # # - +
# # - + #
# # # # #
# # # # #
# # # - +
# # # # #
# # # # #
# # # # -
+ # # # #
# # - + #
# # # # #
# # # # ]
4438 [ # # + - : 676 : /* value= */ value.expr(),
+ - + - #
# # # + -
+ - + - +
- + - + -
# # + - +
- # # # #
# # + - #
# # # # #
+ - # # +
- # # #
# ]
4439 [ # # + - : 676 : /* type= */ wasm_type<pointed_type, L>(),
+ - + - #
# # # + -
+ - + - +
- + - + -
# # + - +
- # # # #
# # + - #
# # # # #
+ - # # +
- # # #
# ]
4440 [ # # + - : 676 : /* memory= */ Module::Get().memory_->name
+ - + - #
# # # + -
+ - + - +
- + - + -
# # + - +
- # # # #
# # + - #
# # # # #
+ - # # +
- # # #
# ]
4441 : : );
4442 : 676 : Module::Block().list.push_back(e);
4443 : : } else {
4444 : 0 : auto vectors = value.vectors();
4445 [ # # # # : 0 : for (std::size_t idx = 0; idx < PrimitiveExpr<pointed_type, L>::num_vectors; ++idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4446 [ # # # # : 0 : auto addr_cpy = addr_.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4447 : 0 : auto offset = offset_ + offset_t(idx * 16);
4448 [ # # # # : 0 : auto e = Module::Builder().makeStore(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4449 : : /* bytes= */ 16,
4450 [ # # # # : 0 : /* offset= */ offset >= 0 ? offset : 0,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4451 : : /* align= */ alignof(pointed_type),
4452 [ # # # # : 0 : /* ptr= */ offset >= 0 ? addr_cpy.expr() : (addr_cpy - uint32_t(-offset)).expr(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4453 [ # # # # : 0 : /* value= */ vectors[idx].expr(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4454 [ # # # # : 0 : /* type= */ ::wasm::Type(::wasm::Type::v128),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4455 [ # # # # : 0 : /* memory= */ Module::Get().memory_->name
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4456 : : );
4457 [ # # # # : 0 : Module::Block().list.push_back(e);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4458 : 0 : }
4459 [ # # # # : 0 : addr_.discard(); // since it was always cloned
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4460 : 0 : }
4461 : 676 : }
4462 : :
4463 : :
4464 : : /*------------------------------------------------------------------------------------------------------------------
4465 : : * Printing
4466 : : *----------------------------------------------------------------------------------------------------------------*/
4467 : :
4468 : : public:
4469 : : friend std::ostream & operator<<(std::ostream &out, const PrimitiveExpr &P) {
4470 : : out << "PrimitiveExpr<" << typeid(T).name() << "*, " << L << ">: " << P.addr_ << " [" << P.offset_ << "]";
4471 : : return out;
4472 : : }
4473 : :
4474 : : void dump(std::ostream &out) const { out << *this << std::endl; }
4475 : : void dump() const { dump(std::cerr); }
4476 : : };
4477 : :
4478 : : namespace detail {
4479 : :
4480 : : template<typename T>
4481 : : struct ptr_helper;
4482 : :
4483 : : template<>
4484 : : struct ptr_helper<void>
4485 : : {
4486 : : using type = PrimitiveExpr<void*, 1>;
4487 : : };
4488 : :
4489 : : template<typename T, std::size_t L>
4490 : : struct ptr_helper<PrimitiveExpr<T, L>>
4491 : : {
4492 : : using type = PrimitiveExpr<T*, L>;
4493 : : };
4494 : :
4495 : : }
4496 : :
4497 : : /** Alias to easily declare `PrimitiveExpr` of pointer to primitive type. */
4498 : : template<typename T>
4499 : : using Ptr = typename detail::ptr_helper<T>::type;
4500 : :
4501 : :
4502 : : /*======================================================================================================================
4503 : : * Expr
4504 : : *====================================================================================================================*/
4505 : :
4506 : : /** An `Expr<T, L>` combines a `PrimitiveExpr<T, L>` value with a `PrimitiveExpr<bool, L>`, called NULL information,
4507 : : * to implement a value with *three-valued logic* (3VL). `Expr<T, L>` provides the same operations as
4508 : : * `PrimitiveExpr<T, L>`. It delegates operations to the underlying value and additionally combines the NULL
4509 : : * information of the operand(s) into the new NULL information of the result. Particular exceptions are `operator
4510 : : * and` and `operator or`, for which `Expr<T, L>` implements 3VL according to [Kleene and Priest's
4511 : : * logic](https://en.wikipedia.org/wiki/Three-valued_logic#Kleene_and_Priest_logics). */
4512 : : template<dsl_primitive T, std::size_t L>
4513 : : requires requires { PrimitiveExpr<T, L>(); PrimitiveExpr<bool, L>(); }
4514 : : struct Expr<T, L>
4515 : : {
4516 : : using type = T;
4517 : : static constexpr std::size_t num_simd_lanes = L;
4518 : : using primitive_type = PrimitiveExpr<T, L>;
4519 : :
4520 : : /*----- Friends --------------------------------------------------------------------------------------------------*/
4521 : : template<typename, std::size_t> friend struct Expr; // to convert Expr<U, L> to Expr<T, L>
4522 : : template<typename, VariableKind, bool, std::size_t> friend class detail::variable_storage; // to use split_unsafe()
4523 : :
4524 : : private:
4525 : : ///> the referenced value expression
4526 : : PrimitiveExpr<T, L> value_;
4527 : : /** A boolean expression that evaluates to `true` at runtime iff this `Expr` is `NULL`.
4528 : : * If this `Expr` cannot be `NULL`, then `is_null_` evaluates to `false` at compile time, i.e. `not is_null_`. */
4529 [ # # # # : 2942 : PrimitiveExpr<bool, L> is_null_ = PrimitiveExpr<bool, L>();
# # # # #
# # # #
# ]
4530 : :
4531 : : public:
4532 : : ///> *Implicitly* constructs an `Expr` from a \p value.
4533 : 2942 : Expr(PrimitiveExpr<T, L> value) : value_(value) {
4534 [ + - + - : 2942 : M_insist(bool(value_), "value must be present");
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # + -
+ - # # #
# # # + -
+ - + - #
# # # # #
# # # # +
- # # # #
# # # # +
- + - + -
# # # # +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4535 : 2942 : }
4536 : :
4537 : : ///> Constructs an `Expr` from a \p value and NULL information \p is_null.
4538 : 5122 : Expr(PrimitiveExpr<T, L> value, PrimitiveExpr<bool, L> is_null)
4539 : 5122 : : value_(value)
4540 [ + - + - : 5122 : , is_null_(is_null)
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # + -
+ - + - #
# # # # #
# # + - +
- # # # #
+ - + - #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4541 : : {
4542 [ + - + - : 5122 : M_insist(bool(value_), "value must be present");
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
+ - + - #
# # # + -
+ - # # +
- + - + -
# # # # #
# # # + -
+ - # # +
- # # # #
# # # # #
# # # + -
# # # # #
# # # # #
# # # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4543 [ + - + - : 5122 : if (is_null)
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
+ - # # +
- + - # #
# # # # #
# # # + -
+ - + - #
# # # # #
+ - + - #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4544 : : M_insist_no_ternary_logic();
4545 : 5122 : }
4546 : :
4547 : : ///> Constructs an `Expr` from a `std::pair` \p value of value and NULL info.
4548 : 3730 : explicit Expr(std::pair<PrimitiveExpr<T, L>, PrimitiveExpr<bool, L>> value)
4549 [ # # # # : 3730 : : Expr(value.first, value.second)
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + # # #
# + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + #
# # # + -
- + + - -
+ + - - +
# # # # #
# # # + -
- + + - -
+ # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4550 : 3730 : { }
4551 : :
4552 : : /** Constructs an `Expr` from a constant \p value. */
4553 : : template<dsl_primitive... Us>
4554 : : requires (sizeof...(Us) > 0) and requires (Us... us) { PrimitiveExpr<T, L>(us...); }
4555 : 1868 : explicit Expr(Us... value)
4556 [ + - + - : 1868 : : Expr(PrimitiveExpr<T, L>(value...))
# # + - +
- + - + -
+ - # # #
# + - + -
+ - + - #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
# # # # #
# # # # #
# # ]
4557 : 1868 : { }
4558 : :
4559 : : /** Constructs an `Expr` from a decayable constant \p value. */
4560 : : template<decayable... Us>
4561 : : requires (sizeof...(Us) > 0) and (dsl_primitive<std::decay_t<Us>> and ...) and
4562 : : requires (Us... us) { Expr(std::decay_t<Us>(us)...); }
4563 : : explicit Expr(Us... value)
4564 : : : Expr(std::decay_t<Us>(value)...)
4565 : : { }
4566 : :
4567 : : Expr(const Expr&) = delete;
4568 : : ///> Constructs a new `Expr` by *moving* the underlying `value_` and `is_null_` of `other` to `this`.
4569 [ + - + - : 2540 : Expr(Expr &other) : Expr(other.split_unsafe()) { /* move, not copy */ }
# # + - +
- + - + -
+ - + - +
- + - + -
# # + - +
- + - + -
+ - + - +
- # # + -
+ - + - #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4570 : : ///> Constructs a new `Expr` by *moving* the underlying `value_` and `is_null_` of `other` to `this`.
4571 [ + - + - : 1190 : Expr(Expr &&other) : Expr(other.split_unsafe()) { }
# # + - +
- + - + -
+ - + - +
- + - + -
# # + - +
- + - + -
+ - + - #
# + - + -
# # + - #
# # # # #
+ - # # #
# # # #
# ]
4572 : :
4573 : : Expr & operator=(Expr&&) = delete;
4574 : :
4575 : 8064 : ~Expr() {
4576 [ + - + - : 8064 : M_insist(not bool(value_), "value must be used or explicitly discarded");
# # # # +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # #
# + - + -
+ - + - #
# # # + -
+ - # # #
# + - + -
+ - + - +
- # # + -
+ - # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4577 [ + - + - : 8064 : M_insist(not bool(is_null_), "NULL flag must be used or explicitly discarded");
# # # # +
- + - + -
+ - + - +
- + - + -
# # + - +
- # # # #
# # + - +
- + - + -
+ - + - #
# # # + -
+ - # # #
# # # + -
+ - # # #
# # # + -
# # # # #
# + - + -
# # + - +
- + - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
4578 : 8064 : }
4579 : :
4580 : : private:
4581 : : /** Splits this `Expr` into a `PrimitiveExpr<T, L>` with the value and a `PrimitiveExpr<bool, L>` with the `NULL`
4582 : : * information. Then, *moves* these `PrimitiveExpr`s out of `this`. Special care must be taken as the NULL
4583 : : * information may be unusable, i.e. missing AST. */
4584 : 4066 : std::pair<PrimitiveExpr<T, L>, PrimitiveExpr<bool, L>> split_unsafe() {
4585 : 4066 : M_insist(bool(value_), "`Expr` has already been moved");
4586 : 4066 : return { value_, is_null_ };
4587 : : }
4588 : :
4589 : : public:
4590 : : /** Returns `true` if this `Expr` actually holds a value (Binaryen AST), `false` otherwise. Can be used to test
4591 : : * whether this `Expr` has already been used. */
4592 : : explicit operator bool() const { return bool(value_); }
4593 : :
4594 : : /** *Moves* the current `value_` out of `this`. Requires (and insists) that `this` cannot be `NULL`. */
4595 : 742 : PrimitiveExpr<T, L> insist_not_null() {
4596 : 742 : M_insist(bool(value_), "`Expr` has already been moved");
4597 [ + - + - : 742 : if (can_be_null())
+ - + - +
- + - + -
+ - # # #
# + - + -
# # + - +
- + - + -
+ - # # +
- + - # #
# # + - #
# # # # #
+ - # # #
# # # #
# ]
4598 [ # # # # : 0 : Wasm_insist(not is_null_, "must not be NULL");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4599 : 742 : return value_;
4600 : 0 : }
4601 : :
4602 : : /** Splits this `Expr` into a `PrimitiveExpr<T, L>` with the value and a `PrimitiveExpr<bool, L>` with the `NULL`
4603 : : * information. Then, *moves* these `PrimitiveExpr`s out of `this`. */
4604 : 0 : std::pair<PrimitiveExpr<T, L>, PrimitiveExpr<bool, L>> split() {
4605 : 0 : M_insist(bool(value_), "`Expr` has already been moved");
4606 : 0 : auto [value, is_null] = split_unsafe();
4607 [ # # # # : 0 : if (is_null)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4608 [ # # # # : 0 : return { value, is_null };
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4609 : : else
4610 [ # # # # : 0 : return { value, PrimitiveExpr<bool, L>(false) };
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4611 : 0 : }
4612 : :
4613 : : /** Returns a *deep copy* of `this`. */
4614 : 230 : Expr clone() const {
4615 : 230 : M_insist(bool(value_), "`Expr` has already been moved`");
4616 [ + - + - : 230 : return Expr(
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
- + # # #
# + - + -
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4617 : 230 : /* value= */ value_.clone(),
4618 [ + - + - : 230 : /* is_null= */ is_null_ ? is_null_.clone() : PrimitiveExpr<bool, L>()
+ - # # #
# # # # #
# # # # #
# + - + -
# # # # #
# # # + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4619 : : );
4620 : 0 : }
4621 : :
4622 : : /** Discards `this`. */
4623 : 68 : void discard() {
4624 : 68 : value_.discard();
4625 [ - + - + : 68 : if (can_be_null())
# # - + -
+ - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4626 : 68 : is_null_.discard();
4627 : 68 : }
4628 : :
4629 : :
4630 : : /*------------------------------------------------------------------------------------------------------------------
4631 : : * methods related to NULL
4632 : : *----------------------------------------------------------------------------------------------------------------*/
4633 : :
4634 : : public:
4635 : : /** Returns `true` if `this` *may be* `NULL`, `false` otherwise. */
4636 : 3340 : bool can_be_null() const { return bool(is_null_); }
4637 : :
4638 : : /** Returns `true` if `this` is `NULL`, `false` otherwise. */
4639 : 248 : PrimitiveExpr<bool, L> is_null() {
4640 : 248 : value_.discard();
4641 [ + - + - : 248 : if (can_be_null()) {
+ - + + +
- + - + +
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4642 : : M_insist_no_ternary_logic();
4643 : 240 : return is_null_;
4644 : : } else {
4645 : 8 : return PrimitiveExpr<bool, L>(false);
4646 : : }
4647 : 248 : }
4648 : :
4649 : : /** Returns `true` if `this` is `NOT NULL`, `false` otherwise. */
4650 : 16 : PrimitiveExpr<bool, L> not_null() {
4651 : 16 : value_.discard();
4652 [ # # # # : 16 : if (can_be_null()) {
# # + + #
# # # + +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4653 : : M_insist_no_ternary_logic();
4654 : 12 : return not is_null_;
4655 : : } else {
4656 : 4 : return PrimitiveExpr<bool, L>(true);
4657 : : }
4658 : 16 : }
4659 : :
4660 : : /** Returns `true` if the value is `true` and `NOT NULL`. Useful to use this `Expr<bool, L>` for conditional
4661 : : * control flow. */
4662 : 286 : PrimitiveExpr<bool, L> is_true_and_not_null() requires boolean<T> {
4663 [ + + - + : 286 : if (can_be_null())
+ - ]
4664 [ - + - + : 198 : return value_ and not is_null_;
# # ]
4665 : : else
4666 : 88 : return value_;
4667 : 286 : }
4668 : :
4669 : : /** Returns `true` if the value is `false` and `NOT NULL`. Useful to use this `Expr<bool, L>` for conditional
4670 : : * control flow. */
4671 : 104 : PrimitiveExpr<bool, L> is_false_and_not_null() requires boolean<T> {
4672 [ + + - + : 104 : if (can_be_null())
+ + ]
4673 [ + - - + : 58 : return not value_ and not is_null_;
+ - - + +
- - + ]
4674 : : else
4675 : 46 : return not value_;
4676 : 104 : }
4677 : :
4678 : :
4679 : : /*------------------------------------------------------------------------------------------------------------------
4680 : : * Factory method for NULL
4681 : : *----------------------------------------------------------------------------------------------------------------*/
4682 : :
4683 : : public:
4684 : : /** Returns an `Expr` that is `NULL`. */
4685 [ + - - + : 204 : static Expr Null() { return Expr(PrimitiveExpr<T, L>(T()), PrimitiveExpr<bool, L>(true)); }
+ - - + #
# # # + -
- + # # #
# # # # #
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4686 : :
4687 : :
4688 : : /*------------------------------------------------------------------------------------------------------------------
4689 : : * Conversion operations
4690 : : *----------------------------------------------------------------------------------------------------------------*/
4691 : :
4692 : : public:
4693 : : /** *Implicitly* converts an `Expr<T, L>` to an `Expr<To, ToL>`. Only applicable if `PrimitiveExpr<T, L>` is
4694 : : * implicitly convertible to `PrimitiveExpr<To, ToL>`. */
4695 : : template<dsl_primitive To, std::size_t ToL = L>
4696 : : requires requires { static_cast<PrimitiveExpr<To, ToL>>(value_); }
4697 [ + - - + : 32 : operator Expr<To, ToL>() { return Expr<To, ToL>(static_cast<PrimitiveExpr<To, ToL>>(value_), is_null_); }
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + ]
4698 : :
4699 : : /** *Explicitly* converts an `Expr<T, L>` to an `Expr<To, ToL>`. Only applicable if `PrimitiveExpr<T, L>` is
4700 : : * explicitly convertible to `PrimitiveExpr<To, ToL>` (via method `to<To, ToL>()`). */
4701 : : template<dsl_primitive To, std::size_t ToL = L>
4702 : : requires requires { value_.template to<To, ToL>(); }
4703 [ + - - + : 62 : Expr<To, ToL> to() { return Expr<To, ToL>(value_.template to<To, ToL>(), is_null_); }
+ - - + #
# # # + -
- + + - -
+ + - - +
# # # # +
- - + # #
# # # # #
# + - - +
# # # # +
- - + + -
- + + - -
+ + - - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4704 : :
4705 : : /** Reinterpret an `Expr<T, L>` to an `Expr<To, ToL>`. Only applicable if `PrimitiveExpr<T, L>` can be
4706 : : * reinterpreted to `PrimitiveExpr<To, ToL>`. */
4707 : : template<dsl_primitive To, std::size_t ToL = L>
4708 : : requires requires { value_.template reinterpret<To, ToL>(); }
4709 : : Expr<To, ToL> reinterpret() { return Expr<To, ToL>(value_.template reinterpret<To, ToL>(), is_null_); }
4710 : :
4711 : : /** Broadcasts a `PrimitiveExpr<T, 1>` to a `PrimitiveExpr<T, ToL>`. Only applicable if `PrimitiveExpr<T, 1>`
4712 : : * can be broadcasted to `PrimitiveExpr<T, ToL>`. */
4713 : : template<std::size_t ToL>
4714 : : requires requires { value_.template broadcast<ToL>(); is_null_.template broadcast<ToL>(); }
4715 : : Expr<T, ToL> broadcast() {
4716 : : return Expr<T, ToL>(value_.template broadcast<ToL>(), is_null_.template broadcast<ToL>());
4717 : : }
4718 : :
4719 : :
4720 : : /*------------------------------------------------------------------------------------------------------------------
4721 : : * Unary operations
4722 : : *----------------------------------------------------------------------------------------------------------------*/
4723 : :
4724 : : public:
4725 : : /** List of supported unary operators on `PrimitiveExpr`. */
4726 : : #define UNARY_LIST(X) \
4727 : : X(make_signed) \
4728 : : X(make_unsigned) \
4729 : : X(operator +) \
4730 : : X(operator -) \
4731 : : X(abs) \
4732 : : X(ceil) \
4733 : : X(floor) \
4734 : : X(trunc) \
4735 : : X(nearest) \
4736 : : X(sqrt) \
4737 : : X(operator ~) \
4738 : : X(clz) \
4739 : : X(ctz) \
4740 : : X(popcnt) \
4741 : : X(eqz) \
4742 : : X(operator not)
4743 : :
4744 : : #define UNARY(OP) \
4745 : : auto OP() requires requires { value_.OP(); } { \
4746 : : using PrimExprT = decltype(value_.OP()); \
4747 : : using ExprT = expr_t<PrimExprT>; \
4748 : : return ExprT(value_.OP(), is_null_); \
4749 : : }
4750 [ # # # # : 100 : UNARY_LIST(UNARY)
# # # # +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + # #
# # # # #
# # # # #
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + # #
# # # # #
# + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - + #
# # # # #
# # ]
4751 : : #undef UNARY
4752 : :
4753 : : /*----- Arithmetical operations with special three-valued logic --------------------------------------------------*/
4754 : :
4755 : 10 : auto add_pairwise() requires requires { value_.add_pairwise(); } {
4756 : : using PrimExprT = decltype(value_.add_pairwise());
4757 : : using ExprT = expr_t<PrimExprT>;
4758 [ + - + - : 10 : if (can_be_null())
+ - + - -
+ ]
4759 [ # # # # : 2 : return ExprT(value_.add_pairwise(), is_null_.template to<uint8_t>().add_pairwise().template to<bool>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - -
+ ]
4760 : : else
4761 [ + - + - : 8 : return ExprT(value_.add_pairwise());
+ - + - #
# ]
4762 : 10 : }
4763 : :
4764 : : /*----- Bitwise operations with special three-valued logic -------------------------------------------------------*/
4765 : :
4766 : 10 : Expr<uint32_t, 1> bitmask() requires requires { value_.bitmask(); } {
4767 [ + - - + ]: 10 : if (can_be_null())
4768 [ # # # # : 8 : return Expr<uint32_t, 1>(value_.bitmask(), is_null_.any_true());
+ - - + ]
4769 : : else
4770 [ + - # # ]: 2 : return Expr<uint32_t, 1>(value_.bitmask());
4771 : 10 : }
4772 : :
4773 : : /*----- Logical operations with special three-valued logic -------------------------------------------------------*/
4774 : :
4775 : 12 : Expr<bool, 1> any_true() requires requires { value_.any_true(); } {
4776 [ + - - + ]: 12 : if (can_be_null())
4777 [ # # # # : 8 : return Expr<bool, 1>(value_.any_true(), is_null_.any_true());
+ - - + ]
4778 : : else
4779 [ + - # # ]: 4 : return Expr<bool, 1>(value_.any_true());
4780 : 12 : }
4781 : :
4782 : 12 : Expr<bool, 1> all_true() requires requires { value_.all_true(); } {
4783 [ + - - + ]: 12 : if (can_be_null())
4784 [ # # # # : 8 : return Expr<bool, 1>(value_.all_true(), is_null_.any_true());
+ - - + ]
4785 : : else
4786 [ + - # # ]: 4 : return Expr<bool, 1>(value_.all_true());
4787 : 12 : }
4788 : :
4789 : : /*----- Hashing operations with special three-valued logic -------------------------------------------------------*/
4790 : :
4791 : 0 : PrimitiveExpr<uint64_t, L> hash() requires requires { value_.hash(); } {
4792 [ # # # # : 0 : if (can_be_null())
# # # # #
# # # #
# ]
4793 [ # # # # : 0 : return Select(is_null_, PrimitiveExpr<uint64_t, 1>(1UL << 63), value_.hash());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4794 : : else
4795 : 0 : return value_.hash();
4796 : 0 : }
4797 : :
4798 : :
4799 : : /*------------------------------------------------------------------------------------------------------------------
4800 : : * Binary operations
4801 : : *----------------------------------------------------------------------------------------------------------------*/
4802 : :
4803 : : public:
4804 : : #define BINARY(OP) \
4805 : : template<dsl_primitive U> \
4806 : : auto OP(Expr<U, L> other) requires requires { this->value_.OP(other.value_); } { \
4807 : : const unsigned idx = (other.can_be_null() << 1U) | this->can_be_null(); \
4808 : : auto result = this->value_.OP(other.value_); \
4809 : : using ReturnType = typename decltype(result)::type; \
4810 : : constexpr std::size_t ReturnLength = decltype(result)::num_simd_lanes; \
4811 : : switch (idx) { \
4812 : : default: M_unreachable("invalid index"); \
4813 : : case 0b00: /* neither `this` nor `other` can be `NULL` */ \
4814 : : return Expr<ReturnType, ReturnLength>(result); \
4815 : : case 0b01: /* `this` can be `NULL` */ \
4816 : : return Expr<ReturnType, ReturnLength>(result, this->is_null_); \
4817 : : case 0b10: /* `other` can be `NULL` */ \
4818 : : return Expr<ReturnType, ReturnLength>(result, other.is_null_); \
4819 : : case 0b11: /* both `this` and `other` can be `NULL` */ \
4820 : : return Expr<ReturnType, ReturnLength>(result, this->is_null_ or other.is_null_); \
4821 : : } \
4822 : : }
4823 : :
4824 [ # # # # : 112 : BINARY(operator +)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - +
+ + + # #
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4825 [ # # # # : 72 : BINARY(operator -)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4826 [ # # # # : 228 : BINARY(operator *)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4827 [ + - - - : 126 : BINARY(operator /)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4828 [ # # # # : 2 : BINARY(operator %)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4829 [ + - - - : 4 : BINARY(operator bitand)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
4830 [ + - - - : 16 : BINARY(operator bitor)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# ]
4831 [ + - - - : 4 : BINARY(operator xor)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # ]
4832 [ + - - - : 2 : BINARY(operator <<)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # ]
4833 [ + - - - : 2 : BINARY(operator >>)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # ]
4834 [ + - - - : 180 : BINARY(operator ==)
+ - - # #
# # # # #
# # # # #
+ - + - +
- # # # #
# # # # +
- - - + -
- # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - - + -
- # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # + - -
- + - - #
# # # # #
# # # # #
# + - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4835 [ # # # # : 14 : BINARY(operator !=)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4836 [ # # # # : 38 : BINARY(operator <)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4837 [ # # # # : 20 : BINARY(operator <=)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4838 [ # # # # : 38 : BINARY(operator >)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4839 [ # # # # : 20 : BINARY(operator >=)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - - -
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
- - - - +
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - - -
- + # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4840 [ + - - - : 4 : BINARY(copy_sign)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # ]
4841 [ + - - - : 6 : BINARY(min)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # #
# ]
4842 [ + - - - : 6 : BINARY(max)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # +
- - - - -
+ # # + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # + - -
- - - + #
# + - + -
# # # # #
# # # # #
# # # # #
# # # #
# ]
4843 [ + - - - : 2 : BINARY(avg)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # ]
4844 [ + - - - : 2 : BINARY(rotl)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # ]
4845 [ + - - - : 2 : BINARY(rotr)
- - + # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # ]
4846 : : #undef BINARY
4847 : :
4848 : : /*----- Bitwise operations with special three-valued logic -------------------------------------------------------*/
4849 : :
4850 : : template<dsl_primitive U>
4851 : 2 : Expr operator<<(Expr<U, 1> other) requires requires { this->value_ << other.value_; } and (L > 1) {
4852 : 2 : const unsigned idx = (bool(other.is_null_) << 1U) | bool(this->is_null_);
4853 [ + - ]: 2 : PrimitiveExpr result = this->value_ << other.value_;
4854 [ - - - - : 2 : switch (idx) {
+ ]
4855 [ # # ]: 0 : default: M_unreachable("invalid index");
4856 : :
4857 : : case 0b00: { /* neither `this` nor `other` can be `NULL` */
4858 [ + - + - ]: 2 : return Expr(result);
4859 : : }
4860 : : case 0b01: { /* `this` can be `NULL` */
4861 [ # # # # : 0 : return Expr(result, this->is_null_);
# # ]
4862 : : }
4863 : : case 0b10: { /* `other` can be `NULL` */
4864 [ # # ]: 0 : PrimitiveExpr<bool, L> is_null = other.is_null_.template broadcast<L>();
4865 [ # # # # : 0 : return Expr(result, is_null);
# # ]
4866 : 0 : }
4867 : : case 0b11: { /* both `this` and `other` can be `NULL` */
4868 [ # # # # ]: 0 : PrimitiveExpr<bool, L> is_null = this->is_null_ or other.is_null_.template broadcast<L>();
4869 [ # # # # : 0 : return Expr(result, is_null);
# # ]
4870 : 0 : }
4871 : : }
4872 : 2 : }
4873 : :
4874 : : template<dsl_primitive U>
4875 : 2 : Expr operator>>(Expr<U, 1> other) requires requires { this->value_ >> other.value_; } and (L > 1) {
4876 : 2 : const unsigned idx = (bool(other.is_null_) << 1U) | bool(this->is_null_);
4877 [ + - ]: 2 : PrimitiveExpr result = this->value_ >> other.value_;
4878 [ - - - - : 2 : switch (idx) {
+ ]
4879 [ # # ]: 0 : default: M_unreachable("invalid index");
4880 : :
4881 : : case 0b00: { /* neither `this` nor `other` can be `NULL` */
4882 [ + - + - ]: 2 : return Expr(result);
4883 : : }
4884 : : case 0b01: { /* `this` can be `NULL` */
4885 [ # # # # : 0 : return Expr(result, this->is_null_);
# # ]
4886 : : }
4887 : : case 0b10: { /* `other` can be `NULL` */
4888 [ # # ]: 0 : PrimitiveExpr<bool, L> is_null = other.is_null_.template broadcast<L>();
4889 [ # # # # : 0 : return Expr(result, is_null);
# # ]
4890 : 0 : }
4891 : : case 0b11: { /* both `this` and `other` can be `NULL` */
4892 [ # # # # ]: 0 : PrimitiveExpr<bool, L> is_null = this->is_null_ or other.is_null_.template broadcast<L>();
4893 [ # # # # : 0 : return Expr(result, is_null);
# # ]
4894 : 0 : }
4895 : : }
4896 : 2 : }
4897 : :
4898 : : /*----- Logical operations with special three-valued logic -------------------------------------------------------*/
4899 : :
4900 : : /** Implements logical *and* according to 3VL of [Kleene and Priest's
4901 : : * logic](https://en.wikipedia.org/wiki/Three-valued_logic#Kleene_and_Priest_logics). */
4902 : 94 : Expr<bool, L> operator and(Expr<bool, L> other) requires requires { this->value_ and other.value_; } {
4903 : 94 : const unsigned idx = (bool(other.is_null_) << 1U) | bool(this->is_null_);
4904 [ + + + + : 94 : switch (idx) {
- # # # #
# # # # #
# - - - +
- ]
4905 : 0 : default: M_unreachable("invalid index");
4906 : :
4907 : : case 0b00: { /* neither `this` nor `other` can be `NULL` */
4908 [ + - # # : 34 : PrimitiveExpr<bool, L> result = this->value_ and other.value_;
# # + - ]
4909 [ + - + - : 34 : return Expr<bool, L>(result);
# # # # #
# # # + -
+ - ]
4910 : 34 : }
4911 : : case 0b01: { /* `this` can be `NULL` */
4912 [ + - # # : 8 : PrimitiveExpr<bool, L> result = this->value_ and other.value_.clone();
# # # # ]
4913 : : PrimitiveExpr<bool, L> is_null =
4914 [ + - # # : 8 : this->is_null_ and // `this` is NULL
# # # # ]
4915 [ + - # # : 8 : other.value_; // `other` does not dominate, i.e. is true
# # # # ]
4916 [ + - + - : 8 : return Expr<bool, L>(result, is_null);
+ - # # #
# # # # #
# # # # #
# # # #
# ]
4917 : 8 : }
4918 : : case 0b10: { /* `other` can be `NULL` */
4919 [ + - + - : 8 : PrimitiveExpr<bool, L> result = this->value_.clone() and other.value_;
# # # # #
# # # # #
# # ]
4920 : : PrimitiveExpr<bool, L> is_null =
4921 [ + - # # : 8 : other.is_null_ and // `other` is NULL
# # # # ]
4922 [ + - # # : 8 : this->value_; // `this` does not dominate, i.e. is true
# # # # ]
4923 [ + - + - : 8 : return Expr<bool, L>(result, is_null);
+ - # # #
# # # # #
# # # # #
# # # #
# ]
4924 : 8 : }
4925 : : case 0b11: { /* both `this` and `other` can be `NULL` */
4926 : 44 : auto this_is_null = this->is_null_.clone();
4927 [ + - # # : 44 : auto other_is_null = other.is_null_.clone();
# # # # ]
4928 [ + - + - : 44 : PrimitiveExpr<bool, L> result = this->value_.clone() and other.value_.clone();
+ - # # #
# # # # #
# # # # #
# # # #
# ]
4929 : : PrimitiveExpr<bool, L> is_null =
4930 [ + - + - : 132 : (this_is_null or other_is_null) and // at least one is NULL
+ - # # #
# # # # #
# # # # #
# # # #
# ]
4931 [ + - + - : 88 : (this->value_ or this->is_null_) and // `this` does not dominate, i.e. is not real false
+ - # # #
# # # # #
# # # # #
# # # #
# ]
4932 [ + - + - : 44 : (other.value_ or other.is_null_); // `other` does not dominate, i.e. is not real false
# # # # #
# # # # #
# # ]
4933 [ + - + - : 44 : return Expr<bool, L>(result, is_null);
- + # # #
# # # # #
# # # # #
# # # #
# ]
4934 : 44 : }
4935 : : }
4936 : 94 : }
4937 : :
4938 : : /** Implements logical *and not* according to 3VL of [Kleene and Priest's
4939 : : * logic](https://en.wikipedia.org/wiki/Three-valued_logic#Kleene_and_Priest_logics). */
4940 : 44 : Expr<bool, L> and_not(Expr<bool, L> other) requires requires { this->value_.and_not(other.value_); } {
4941 : 44 : const unsigned idx = (bool(other.is_null_) << 1U) | bool(this->is_null_);
4942 [ - - - + : 44 : switch (idx) {
- + + + +
- ]
4943 : 0 : default: M_unreachable("invalid index");
4944 : :
4945 : : case 0b00: { /* neither `this` nor `other` can be `NULL` */
4946 [ + - + - ]: 26 : PrimitiveExpr<bool, L> result = this->value_.and_not(other.value_);
4947 [ + - + - : 26 : return Expr<bool, L>(result);
+ - + - ]
4948 : 26 : }
4949 : : case 0b01: { /* `this` can be `NULL` */
4950 [ # # + - ]: 8 : PrimitiveExpr<bool, L> result = this->value_.and_not(other.value_.clone());
4951 : : PrimitiveExpr<bool, L> is_null =
4952 [ # # + - ]: 8 : this->is_null_.and_not( // `this` is NULL
4953 [ # # + - ]: 8 : other.value_ // `other` does not dominate, i.e. is false
4954 : : );
4955 [ # # # # : 8 : return Expr<bool, L>(result, is_null);
# # + - +
- + - ]
4956 : 8 : }
4957 : : case 0b10: { /* `other` can be `NULL` */
4958 [ # # # # : 8 : PrimitiveExpr<bool, L> result = this->value_.clone().and_not(other.value_);
+ - + - ]
4959 : : PrimitiveExpr<bool, L> is_null =
4960 [ # # + - ]: 8 : other.is_null_ and // `other` is NULL
4961 [ # # + - ]: 8 : this->value_; // `this` does not dominate, i.e. is true
4962 [ # # # # : 8 : return Expr<bool, L>(result, is_null);
# # + - +
- + - ]
4963 : 8 : }
4964 : : case 0b11: { /* both `this` and `other` can be `NULL` */
4965 : 2 : auto this_is_null = this->is_null_.clone();
4966 [ # # + - ]: 2 : auto other_is_null = other.is_null_.clone();
4967 [ # # # # : 2 : PrimitiveExpr<bool, L> result = this->value_.clone().and_not(other.value_.clone());
# # + - +
- + - ]
4968 : : PrimitiveExpr<bool, L> is_null =
4969 [ # # # # : 6 : (this_is_null or other_is_null) and // at least one is NULL
# # + - +
- + - ]
4970 [ # # # # : 4 : (this->value_ or this->is_null_) and // `this` does not dominate, i.e. is not real false
# # + - +
- + - ]
4971 [ # # # # : 2 : (not other.value_ or other.is_null_); // `other` does not dominate, i.e. is not real true
# # + - +
- + - ]
4972 [ # # # # : 2 : return Expr<bool, L>(result, is_null);
# # + - +
- - + ]
4973 : 2 : }
4974 : : }
4975 : 44 : }
4976 : :
4977 : : /** Implements logical *or* according to 3VL of [Kleene and Priest's
4978 : : * logic](https://en.wikipedia.org/wiki/Three-valued_logic#Kleene_and_Priest_logics). */
4979 : 94 : Expr<bool, L> operator or(Expr<bool, L> other) requires requires { this->value_ or other.value_; } {
4980 : 94 : const unsigned idx = (bool(other.is_null_) << 1U) | bool(this->is_null_);
4981 [ + + + + : 94 : switch (idx) {
- - - - +
- # # # #
# ]
4982 : 0 : default: M_unreachable("invalid index");
4983 : :
4984 : : case 0b00: { /* neither `this` nor `other` can be `NULL` */
4985 [ + - + - : 34 : PrimitiveExpr<bool, L> result = this->value_ or other.value_;
# # ]
4986 [ + - + - : 34 : return Expr<bool, L>(result);
+ - + - #
# # # ]
4987 : 34 : }
4988 : : case 0b01: { /* `this` can be `NULL` */
4989 [ + - # # : 8 : PrimitiveExpr<bool, L> result = this->value_ or other.value_.clone();
# # ]
4990 : : PrimitiveExpr<bool, L> is_null =
4991 [ + - # # : 8 : this->is_null_ and // `this` is NULL
# # ]
4992 [ + - # # : 8 : not other.value_; // `other` does not dominate, i.e. is false
# # ]
4993 [ + - + - : 8 : return Expr<bool, L>(result, is_null);
+ - # # #
# # # # #
# # # # ]
4994 : 8 : }
4995 : : case 0b10: { /* `other` can be `NULL` */
4996 [ + - + - : 8 : PrimitiveExpr<bool, L> result = this->value_.clone() or other.value_;
# # # # #
# # # ]
4997 : : PrimitiveExpr<bool, L> is_null =
4998 [ + - # # : 8 : other.is_null_ and // `other` is NULL
# # ]
4999 [ + - # # : 8 : not this->value_; // `this` does not dominate, i.e. is false
# # ]
5000 [ + - + - : 8 : return Expr<bool, L>(result, is_null);
+ - # # #
# # # # #
# # # # ]
5001 : 8 : }
5002 : : case 0b11: { /* both `this` and `other` can be `NULL` */
5003 : 44 : auto this_is_null = this->is_null_.clone();
5004 [ + - # # : 44 : auto other_is_null = other.is_null_.clone();
# # ]
5005 [ + - + - : 44 : PrimitiveExpr<bool, L> result = this->value_.clone() or other.value_.clone();
+ - # # #
# # # # #
# # # # ]
5006 : : PrimitiveExpr<bool, L> is_null =
5007 [ + - + - : 132 : (this_is_null or other_is_null) and // at least one is NULL
+ - # # #
# # # # #
# # # # ]
5008 [ + - + - : 88 : (not this->value_ or this->is_null_) and // `this` does not dominate, i.e. is not real true
+ - + - #
# # # # #
# # # # #
# # # #
# ]
5009 [ + - + - : 44 : (not other.value_ or other.is_null_); // `other` does not dominate, i.e. is not real true
+ - # # #
# # # # #
# # # # ]
5010 [ + - + - : 44 : return Expr<bool, L>(result, is_null);
- + # # #
# # # # #
# # # # ]
5011 : 44 : }
5012 : : }
5013 : 94 : }
5014 : :
5015 : :
5016 : : /*------------------------------------------------------------------------------------------------------------------
5017 : : * Modifications
5018 : : *----------------------------------------------------------------------------------------------------------------*/
5019 : :
5020 : : /** Extracts the \tparam M -th value of `this`. */
5021 : : template<std::size_t M>
5022 : 16 : Expr<T, 1> extract() requires requires { value_.template extract<M>(); } {
5023 [ + - + - : 16 : if (can_be_null())
+ - + - ]
5024 [ # # # # : 0 : return Expr<T, 1>(value_.template extract<M>(), is_null_.template extract<M>());
# # # # #
# # # # #
# # ]
5025 : : else
5026 [ + - + - : 16 : return Expr<T, 1>(value_.template extract<M>());
+ - + - ]
5027 : 16 : }
5028 : :
5029 : : /** Replaces the \tparam M -th value of `this` with \p u. */
5030 : : template<std::size_t M, expr_convertible U>
5031 : : requires requires (expr_t<U> u) { Expr<T, 1>(u); }
5032 : 2 : Expr replace(U &&_value) requires requires (Expr<T, 1> e) { this->value_.template replace<M>(e.value_); } {
5033 : 2 : Expr<T, 1> value(expr_t<U>(std::forward<U>(_value)));
5034 [ - + ]: 2 : if (can_be_null()) {
5035 [ # # ]: 0 : auto [value_val, value_is_null] = value.split();
5036 [ # # # # : 0 : return Expr(value_.template replace<M>(value_val), is_null_.template replace<M>(value_is_null));
# # # # #
# ]
5037 : 0 : } else {
5038 [ + - ]: 2 : M_insist(not value.can_be_null(), "cannot replace a non-nullable value with a nullable one");
5039 [ + - + - ]: 2 : return Expr(value_.template replace<M>(value.value_));
5040 : : }
5041 : 2 : }
5042 : :
5043 : : /** Selects lanes of `this` in byte granularity depending on the indices specified by \p indices. Indices `i` in
5044 : : * the range [0, 15] select the `i`-th` lane, indices outside of this range result in the value 0. */
5045 : : Expr swizzle_bytes(PrimitiveExpr<uint8_t, 16> indices)
5046 : : requires (sizeof(T) == 1) and requires { value_.swizzle_bytes(indices); } {
5047 : : if (can_be_null()) {
5048 : : auto indices_cpy = indices.clone();
5049 : : return Expr(value_.swizzle_bytes(indices), is_null_.swizzle_bytes(indices_cpy));
5050 : : } else {
5051 : : return Expr(value_.swizzle_bytes(indices));
5052 : : }
5053 : : }
5054 : : /** Selects lanes of `this` in byte granularity depending on the indices specified by \p indices. Indices `i` in
5055 : : * the range [0, L * sizeof(T)) select the `i`-th` lane, indices outside of this range result in the value 0. */
5056 : : template<std::size_t M>
5057 : : Expr<T, M / sizeof(T)> swizzle_bytes(const std::array<uint8_t, M> &indices)
5058 : : requires (sizeof(T) == 1) and requires { value_.swizzle_bytes(indices); } {
5059 : : if (can_be_null())
5060 : : return Expr<T, M / sizeof(T)>(value_.swizzle_bytes(indices), is_null_.swizzle_bytes(indices));
5061 : : else
5062 : : return Expr<T, M / sizeof(T)>(value_.swizzle_bytes(indices));
5063 : : }
5064 : :
5065 : : /** Selects lanes of `this` in lane granularity depending on the indices specified by \p indices. Indices `i` in
5066 : : * the range [0, L) select the `i`-th` lane, indices outside of this range result in the value 0. */
5067 : : template<std::size_t M>
5068 : 4 : Expr<T, M> swizzle_lanes(const std::array<uint8_t, M> &indices)
5069 : : requires requires { value_.swizzle_lanes(indices); } {
5070 [ + - + - ]: 4 : if (can_be_null())
5071 [ # # # # : 0 : return Expr<T, M>(value_.swizzle_lanes(indices), is_null_.swizzle_lanes(indices));
# # # # ]
5072 : : else
5073 [ + - + - ]: 4 : return Expr<T, M>(value_.swizzle_lanes(indices));
5074 : 4 : }
5075 : :
5076 : :
5077 : :
5078 : : /*------------------------------------------------------------------------------------------------------------------
5079 : : * Printing
5080 : : *----------------------------------------------------------------------------------------------------------------*/
5081 : :
5082 : : public:
5083 : : friend std::ostream & operator<<(std::ostream &out, const Expr &E) {
5084 : : out << "Expr<" << typeid(type).name() << "," << num_simd_lanes << ">: value_=" << E.value_
5085 : : << ", is_null_=" << E.is_null_;
5086 : : return out;
5087 : : }
5088 : :
5089 : : void dump(std::ostream &out) const { out << *this << std::endl; }
5090 : : void dump() const { dump(std::cerr); }
5091 : : };
5092 : :
5093 : : /** CTAD guide for `Expr` */
5094 : : template<typename T, std::size_t L>
5095 : : Expr(PrimitiveExpr<T, L>, PrimitiveExpr<bool, L>) -> Expr<T, L>;
5096 : :
5097 : : /*----- Forward binary operators on operands convertible to Expr<T, L> -----------------------------------------------*/
5098 : : #define MAKE_BINARY(OP) \
5099 : : template<expr_convertible T, expr_convertible U> \
5100 : : requires (not primitive_convertible<T> or not primitive_convertible<U>) and \
5101 : : requires (expr_t<T> t, expr_t<U> u) { t.OP(u); } \
5102 : : auto OP(T &&t, U &&u) \
5103 : : { \
5104 : : return expr_t<T>(std::forward<T>(t)).OP(expr_t<U>(std::forward<U>(u))); \
5105 : : }
5106 [ + - - + : 558 : BINARY_LIST(MAKE_BINARY)
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ - + - -
+ + - + -
- + + - -
+ + - + -
- + + - +
- - + + -
+ - - + +
- + - - +
+ - - + +
- + - - +
+ - + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
# # # # #
# # # # #
# # + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ ]
5107 : : #undef MAKE_BINARY
5108 : :
5109 : : /*----- Short aliases for all `PrimitiveExpr` and `Expr` types. ------------------------------------------------------*/
5110 : : #define USING_N(TYPE, LENGTH, NAME) \
5111 : : using NAME = PrimitiveExpr<TYPE, LENGTH>; \
5112 : : using _ ## NAME = Expr<TYPE, LENGTH>;
5113 : : #define USING(TYPE, NAME) \
5114 : : template<std::size_t L> using NAME = PrimitiveExpr<TYPE, L>; \
5115 : : template<std::size_t L> using _ ## NAME = Expr<TYPE, L>; \
5116 : : USING_N(TYPE, 1, NAME ## x1) \
5117 : : USING_N(TYPE, 2, NAME ## x2) \
5118 : : USING_N(TYPE, 4, NAME ## x4) \
5119 : : USING_N(TYPE, 8, NAME ## x8) \
5120 : : USING_N(TYPE, 16, NAME ## x16) \
5121 : : USING_N(TYPE, 32, NAME ## x32)
5122 : :
5123 : : USING(bool, Bool)
5124 : : USING(int8_t, I8)
5125 : : USING(uint8_t, U8)
5126 : : USING(int16_t, I16)
5127 : : USING(uint16_t, U16)
5128 : : USING(int32_t, I32)
5129 : : USING(uint32_t, U32)
5130 : : USING(int64_t, I64)
5131 : : USING(uint64_t, U64)
5132 : : USING(float, Float)
5133 : : USING(double, Double)
5134 : : ///> this is neither signed nor unsigned char (see https://en.cppreference.com/w/cpp/language/types, Character types)
5135 : : USING_N(char, 1, Charx1)
5136 : : USING_N(char, 16, Charx16)
5137 : : USING_N(char, 32, Charx32)
5138 : : #undef USING
5139 : :
5140 : :
5141 : : /*======================================================================================================================
5142 : : * Variable
5143 : : *====================================================================================================================*/
5144 : :
5145 : : namespace detail {
5146 : :
5147 : : /** Allocates a fresh local variable of type \tparam T and number of SIMD lanes \tparam L in the currently active
5148 : : * function's stack and returns the variable's `::wasm::Index`. */
5149 : : template<dsl_primitive T, std::size_t L>
5150 : 1850 : ::wasm::Index allocate_local()
5151 : : {
5152 : 1850 : ::wasm::Function &fn = Module::Function();
5153 : 1850 : const ::wasm::Index index = fn.getNumParams() + fn.vars.size();
5154 : 1850 : const ::wasm::Type type = wasm_type<T, L>();
5155 : 1850 : fn.vars.emplace_back(type); // allocate new local variable
5156 : 1850 : M_insist(fn.isVar(index));
5157 : 1850 : M_insist(fn.getLocalType(index) == type);
5158 : 1850 : return index;
5159 : : }
5160 : :
5161 : : /** Helper class to select the appropriate storage for a `Variable`. Local variables are allocated on the currently
5162 : : * active function's stack whereas global variables are allocated globally. Local variables of primitive type
5163 : : * can have an additional `NULL` information. */
5164 : : template<typename T, VariableKind Kind, bool CanBeNull, std::size_t L>
5165 : : class variable_storage;
5166 : :
5167 : : /** Specialization for local variables of arithmetic type that *cannot* be `NULL`. */
5168 : : template<dsl_primitive T, VariableKind Kind, std::size_t L>
5169 : : requires arithmetic<T> or (boolean<T> and Kind == VariableKind::Param)
5170 : : class variable_storage<T, Kind, /* CanBeNull= */ false, L>
5171 : : {
5172 : : ///> the number of Wasm locals needed
5173 : : static constexpr std::size_t num_locals = ((L * sizeof(T)) + 15) / 16;
5174 : : static_assert(Kind != VariableKind::Param or num_locals == 1, "parameters must fit in a single Wasm local");
5175 : :
5176 : : template<typename, VariableKind, bool, std::size_t>
5177 : : friend class variable_storage; // to enable use in other `variable_storage`
5178 : : friend struct Variable<T, Kind, false, L>; // to be usable by the respective Variable
5179 : :
5180 : : std::array<::wasm::Index, num_locals> indices_; ///< the indices of the local(s)
5181 : : ::wasm::Type type_; ///< the type of the local(s)
5182 : :
5183 : : /** Default-construct. */
5184 : 1850 : variable_storage()
5185 : 3700 : : indices_([](){
5186 : : std::array<::wasm::Index, num_locals> indices;
5187 [ + + + + : 3700 : for (std::size_t idx = 0; idx < num_locals; ++idx)
# # + + +
+ + + # #
+ + + + +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + +
# # # # +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5188 : 1850 : indices[idx] = allocate_local<T, L>();
5189 : 1850 : return indices;
5190 : : }())
5191 : 1850 : , type_(wasm_type<T, L>())
5192 : 1850 : { }
5193 : :
5194 : : variable_storage(const variable_storage&) = delete;
5195 : : variable_storage(variable_storage&&) = default;
5196 : : variable_storage & operator=(variable_storage&&) = default;
5197 : :
5198 : : /** Construct from `::wasm::Index` of already allocated local. */
5199 : 647 : variable_storage(::wasm::Index idx, tag<int>)
5200 : : requires (num_locals == 1)
5201 : 647 : : indices_(std::to_array({ idx })), type_(wasm_type<T, L>())
5202 : : {
5203 : : #ifndef NDEBUG
5204 : 647 : ::wasm::Function &fn = Module::Function();
5205 : 647 : M_insist(fn.isParam(indices_[0]));
5206 : 647 : M_insist(fn.getLocalType(indices_[0]) == type_);
5207 : : #endif
5208 : 647 : }
5209 : :
5210 : : /** Construct from value. */
5211 : : template<typename... Us>
5212 : : requires (sizeof...(Us) > 0) and requires (Us... us) { PrimitiveExpr<T, L>(us...); }
5213 [ + - + - : 58 : explicit variable_storage(Us... value) : variable_storage() { operator=(PrimitiveExpr<T, L>(value...)); }
# # ]
5214 : :
5215 : : /** Construct from value. */
5216 : : template<primitive_convertible U>
5217 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5218 : 554 : explicit variable_storage(U &&value) : variable_storage() { operator=(std::forward<U>(value)); }
5219 : :
5220 : : /** Assign value. */
5221 : : template<primitive_convertible U>
5222 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5223 : 2986 : void operator=(U &&_value) requires (num_locals == 1) {
5224 : 2986 : PrimitiveExpr<T, L> value(primitive_expr_t<U>(std::forward<U>(_value)));
5225 [ + - + - : 2986 : Module::Block().list.push_back(Module::Builder().makeLocalSet(indices_[0], value.expr()));
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5226 : 2986 : }
5227 : : /** Assign value. */
5228 : : template<primitive_convertible U>
5229 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5230 : 0 : void operator=(U &&_value) requires (num_locals > 1) {
5231 : 0 : PrimitiveExpr<T, L> value(primitive_expr_t<U>(std::forward<U>(_value)));
5232 : : static_assert(num_locals == decltype(value)::num_vectors);
5233 [ # # # # : 0 : auto vectors = value.vectors();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5234 [ # # # # : 0 : for (std::size_t idx = 0; idx < num_locals; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5235 [ # # # # : 0 : Module::Block().list.push_back(Module::Builder().makeLocalSet(indices_[idx], vectors[idx].expr()));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5236 : 0 : }
5237 : :
5238 : : /** Retrieve value. */
5239 : 6047 : operator PrimitiveExpr<T, L>() const requires (num_locals == 1) {
5240 [ + - + - : 6047 : return PrimitiveExpr<T, L>(Module::Builder().makeLocalGet(indices_[0], type_));
+ - + - +
- + - # #
+ - + - +
- + - + -
+ - # # +
- # # # #
+ - # # #
# # # + -
# # ]
5241 : 0 : }
5242 : : /** Retrieve value. */
5243 : 0 : operator PrimitiveExpr<T, L>() const requires (num_locals > 1) {
5244 : : static_assert(num_locals == PrimitiveExpr<T, L>::num_vectors);
5245 : 0 : std::array<typename PrimitiveExpr<T, L>::vector_type, num_locals> vectors;
5246 [ # # # # : 0 : for (std::size_t idx = 0; idx < num_locals; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5247 [ # # # # : 0 : vectors[idx] = typename PrimitiveExpr<T, L>::vector_type(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5248 [ # # # # : 0 : Module::Builder().makeLocalGet(indices_[idx], type_)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5249 : : );
5250 [ # # # # : 0 : return PrimitiveExpr<T, L>(std::move(vectors));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5251 : 0 : }
5252 : : };
5253 : :
5254 : : /** Specialization for local variables of boolean type that *cannot* be `NULL`. */
5255 : : template<std::size_t L>
5256 : : class variable_storage<bool, VariableKind::Local, /* CanBeNull= */ false, L>
5257 : : {
5258 : : ///> the number of SIMD lanes of each local bit
5259 : : static constexpr std::size_t bit_num_simd_lanes = std::min<std::size_t>(L, 16);
5260 : : ///> the number of local bits (with maximal number of SIMD lanes of 16) needed
5261 : : static constexpr std::size_t num_locals = (L + 15) / 16;
5262 : :
5263 : : template<typename, VariableKind, bool, std::size_t>
5264 : : friend class variable_storage; // to enable use in other `variable_storage`
5265 : : friend struct Variable<bool, VariableKind::Local, false, L>; // to be usable by the respective Variable
5266 : :
5267 : : ///> stores each boolean value in a single bit
5268 : : std::array<std::shared_ptr<LocalBit<bit_num_simd_lanes>>, num_locals> values_;
5269 : :
5270 : : /** Default-construct. */
5271 : : variable_storage(); // impl delayed because `LocalBit` defined later
5272 : :
5273 : : variable_storage(const variable_storage&) = delete;
5274 : 0 : variable_storage(variable_storage&&) = default;
5275 : : variable_storage & operator=(variable_storage&&) = default;
5276 : :
5277 : : /** Construct from value. */
5278 : : template<typename... Us>
5279 : : requires (sizeof...(Us) > 0) and requires (Us... us) { PrimitiveExpr<bool, L>(us...); }
5280 [ + - - + : 20 : explicit variable_storage(Us... value) : variable_storage() { operator=(PrimitiveExpr<bool, L>(value...)); }
+ - - + +
- - + + -
- + ]
5281 : :
5282 : : /** Construct from value. */
5283 : : template<primitive_convertible U>
5284 : : requires requires (U &&u) { PrimitiveExpr<bool, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5285 [ + - + - : 116 : explicit variable_storage(U &&value) : variable_storage() { operator=(std::forward<U>(value)); }
# # # # #
# # # # #
# # # # #
# ]
5286 : :
5287 : : /** Assign value. */
5288 : : template<primitive_convertible U>
5289 : : requires requires (U &&u) { PrimitiveExpr<bool, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5290 : : void operator=(U &&value); // impl delayed because `LocalBit` defined later
5291 : :
5292 : : /* Set this value to `true`. */
5293 : : void set_true();
5294 : :
5295 : : /* Set this value to `false`. */
5296 : : void set_false();
5297 : :
5298 : : /** Retrieve value. */
5299 : : operator PrimitiveExpr<bool, L>() const; // impl delayed because `LocalBit` defined later
5300 : : };
5301 : :
5302 : : /** Specialization for local variables of primitive type (arithmetic and boolean) that *can* be `NULL`. */
5303 : : template<dsl_primitive T, std::size_t L>
5304 : : class variable_storage<T, VariableKind::Local, /* CanBeNull= */ true, L>
5305 : : {
5306 : : friend struct Variable<T, VariableKind::Local, true, L>; // to be usable by the respective Variable
5307 : :
5308 : : variable_storage<T, VariableKind::Local, false, L> value_;
5309 : : variable_storage<bool, VariableKind::Local, false, L> is_null_;
5310 : :
5311 : : /** Default-construct. */
5312 [ + - # # : 308 : variable_storage() { M_insist_no_ternary_logic(); }
# # ]
5313 : :
5314 : : /** Construct from value. */
5315 : : template<typename... Us>
5316 : : requires (sizeof...(Us) > 0) and requires (Us... us) { Expr<T, L>(us...); }
5317 [ + - + - : 8 : explicit variable_storage(Us... value) : variable_storage() { operator=(Expr<T, L>(value...)); }
+ - - + ]
5318 : :
5319 : : /** Construct from value. */
5320 : : template<expr_convertible U>
5321 : : requires requires (U &&u) { Expr<T, L>(expr_t<U>(std::forward<U>(u))); }
5322 [ + - + - : 300 : explicit variable_storage(U &&value) : variable_storage() { operator=(std::forward<U>(value)); }
+ - + - +
- + - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5323 : :
5324 : : /** Assign value. */
5325 : : template<expr_convertible U>
5326 : : requires requires (U &&u) { Expr<T, L>(expr_t<U>(std::forward<U>(u))); }
5327 : 336 : void operator=(U &&_value) {
5328 [ + - # # ]: 336 : Expr<T, L> value(expr_t<U>(std::forward<U>(_value)));
5329 [ # # + - : 336 : auto [val, is_null] = value.split_unsafe();
# # + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5330 [ # # + - : 336 : this->value_ = val;
# # + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5331 [ # # # # : 336 : this->is_null_ = bool(is_null) ? is_null : PrimitiveExpr<bool, L>(false);
# # # # -
+ # # + -
- + # # #
# # # # #
- + # # +
- - + + -
+ - # # -
+ - + # #
+ - - + +
- + - # #
- + + - +
- # # - +
+ - + - #
# - + + -
+ - # # -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5332 : 336 : }
5333 : :
5334 : : /** Set to `true`. */
5335 : : void set_true()
5336 : : requires boolean<T>
5337 : : {
5338 : : value_.set_true();
5339 : : is_null_.set_false();
5340 : : }
5341 : :
5342 : : /** Set to `false`. */
5343 : : void set_false()
5344 : : requires boolean<T>
5345 : : {
5346 : : value_.set_false();
5347 : : is_null_.set_false();
5348 : : }
5349 : :
5350 : : /** Set to `NULL`. */
5351 : : void set_null() { is_null_.set_true(); }
5352 : :
5353 : : /** Retrieve value. */
5354 [ + - - + : 324 : operator Expr<T, L>() const { return Expr<T, L>(PrimitiveExpr<T, L>(value_), PrimitiveExpr<bool, L>(is_null_)); }
+ - - + #
# # # + -
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - - + ]
5355 : : };
5356 : :
5357 : : /** Specialization for local variables of pointer to primitive type. Pointers *cannot* be `NULL`. */
5358 : : template<dsl_pointer_to_primitive T, VariableKind Kind, std::size_t L>
5359 : : requires (Kind != VariableKind::Global)
5360 : : class variable_storage<T, Kind, /* CanBeNull= */ false, L>
5361 : : {
5362 : : friend struct Variable<T, Kind, false, L>; // to be usable by the respective Variable
5363 : :
5364 : : ///> the address
5365 : : variable_storage<uint32_t, Kind, false, 1> addr_;
5366 : :
5367 : : /** Default-construct. */
5368 : 754 : variable_storage() = default;
5369 : :
5370 : : /** Construct from `::wasm::Index` of already allocated local. */
5371 : 126 : explicit variable_storage(::wasm::Index idx, tag<int> tag) : addr_(idx, tag) { }
5372 : :
5373 : : /** Construct from pointer. */
5374 : : template<primitive_convertible U>
5375 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5376 : 634 : explicit variable_storage(U &&value) : variable_storage() { operator=(std::forward<U>(value)); }
5377 : :
5378 : : /** Assign pointer. */
5379 : : template<primitive_convertible U>
5380 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5381 : 1012 : void operator=(U &&value) {
5382 [ + - - + : 1012 : addr_ = PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(value))).template to<uint32_t, 1>();
+ - - + +
- - + + -
- + + - -
+ # # # #
# # # # +
- - + ]
5383 : 1012 : }
5384 : :
5385 : : /** Retrieve pointer. */
5386 [ + - + - : 2136 : operator PrimitiveExpr<T, L>() const { return PrimitiveExpr<uint32_t, 1>(addr_).template to<T, L>(); }
+ - + - +
- + - # #
+ - ]
5387 : : };
5388 : :
5389 : : /** Specialization for global variables of primitive or pointer to primitive type \tparam T. Global variables must be
5390 : : * of primitive or pointer to primitive type and *cannot* be `NULL`. */
5391 : : template<typename T, std::size_t L>
5392 : : requires dsl_primitive<T> or dsl_pointer_to_primitive<T>
5393 : : class variable_storage<T, VariableKind::Global, /* CanBeNull= */ false, L>
5394 : : {
5395 : : ///> the number of Wasm globals needed; pointers are always stored as single 32-bit unsigned integer global
5396 : : static constexpr std::size_t num_globals = dsl_primitive<T> ? ((L * sizeof(T)) + 15) / 16 : 1;
5397 : :
5398 : : friend struct Variable<T, VariableKind::Global, false, L>; // to be usable by the respective Variable
5399 : :
5400 : : std::array<::wasm::Name, num_globals> names_; ///< the unique names of the global(s)
5401 : : ::wasm::Type type_; ///< the type of the global(s)
5402 : :
5403 : : /** Default construct. */
5404 : 5042 : variable_storage()
5405 : : requires dsl_primitive<T>
5406 : 5042 : : variable_storage(T())
5407 : 5042 : { }
5408 : :
5409 : : variable_storage(const variable_storage&) = delete;
5410 : : variable_storage(variable_storage&&) = default;
5411 : : variable_storage & operator=(variable_storage&&) = default;
5412 : :
5413 : : /** Construct with initial value. */
5414 : : template<typename... Us>
5415 : : requires (sizeof...(Us) > 0) and requires (Us... us) { Module::Get().emit_global<T, L>(names_, true, us...); }
5416 : 5046 : explicit variable_storage(Us... init)
5417 : : requires dsl_primitive<T>
5418 : 10092 : : names_([](){
5419 : 5046 : std::array<::wasm::Name, num_globals> names;
5420 [ + + # # : 10092 : for (std::size_t idx = 0; idx < num_globals; ++idx)
# # # # #
# + + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5421 [ + - - + : 5046 : names[idx] = Module::Unique_Global_Name();
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# - + - +
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5422 : 5046 : return names;
5423 : 0 : }())
5424 : 5046 : , type_(wasm_type<T, L>())
5425 : : {
5426 : 5046 : Module::Get().emit_global<T, L>(names_, true, init...);
5427 : 5046 : }
5428 : : /** Construct with optional initial value. */
5429 : 0 : explicit variable_storage(uint32_t init = 0)
5430 : : requires dsl_pointer_to_primitive<T>
5431 [ # # # # : 0 : : names_(std::to_array<::wasm::Name>({ Module::Unique_Global_Name() }))
# # ]
5432 : 0 : , type_(wasm_type<T, L>())
5433 : : {
5434 : 0 : Module::Get().emit_global<T, L>(names_[0], true, init);
5435 : 0 : }
5436 : :
5437 : : /** Sets the initial value. */
5438 : : template<dsl_primitive... Us>
5439 : : requires (sizeof...(Us) > 0) and requires (Us... us) { make_literal<T, L>(us...); }
5440 : : void init(Us... init) requires dsl_primitive<T> and (num_globals == 1) {
5441 : : Module::Get().module_.getGlobal(names_[0])->init = Module::Builder().makeConst(make_literal<T, L>(init...));
5442 : : }
5443 : : /** Sets the initial value. */
5444 : : template<dsl_primitive... Us>
5445 : : requires (sizeof...(Us) > 0) and
5446 : : requires (Us... us) { { make_literal<T, L>(us...) } -> std::same_as<std::array<::wasm::Literal, num_globals>>; }
5447 : : void init(Us... init) requires dsl_primitive<T> and (num_globals > 1) {
5448 : : auto literals = make_literal<T, L>(init...);
5449 : : for (std::size_t idx = 0; idx < num_globals; ++idx)
5450 : : Module::Get().module_.getGlobal(names_[idx])->init = Module::Builder().makeConst(literals[idx]);
5451 : : }
5452 : : /** Sets the initial value. */
5453 : 0 : void init(uint32_t init) requires dsl_pointer_to_primitive<T> {
5454 [ # # # # : 0 : Module::Get().module_.getGlobal(names_[0])->init = Module::Builder().makeConst(::wasm::Literal(init));
# # ]
5455 : 0 : }
5456 : : /** Sets the initial value. */
5457 : : template<primitive_convertible U>
5458 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5459 : 1654 : void init(U &&_init) requires (num_globals == 1) {
5460 : 1654 : PrimitiveExpr<T, L> init(primitive_expr_t<U>(std::forward<U>(_init)));
5461 [ + - + - : 1654 : Module::Get().module_.getGlobal(names_[0])->init = init.expr();
+ - ]
5462 : 1654 : }
5463 : :
5464 : : /** Assign value. */
5465 : : template<primitive_convertible U>
5466 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5467 : 652 : void operator=(U &&_value) requires (num_globals == 1) {
5468 : 652 : PrimitiveExpr<T, L> value(primitive_expr_t<U>(std::forward<U>(_value)));
5469 [ + - + - : 652 : Module::Block().list.push_back(Module::Builder().makeGlobalSet(names_[0], value.expr()));
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5470 : 652 : }
5471 : : /** Assign value. */
5472 : : template<primitive_convertible U>
5473 : : requires requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5474 : 0 : void operator=(U &&_value) requires (num_globals > 1) {
5475 : 0 : PrimitiveExpr<T, L> value(primitive_expr_t<U>(std::forward<U>(_value)));
5476 : : static_assert(num_globals == decltype(value)::num_vectors);
5477 [ # # # # : 0 : auto vectors = value.vectors();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5478 [ # # # # : 0 : for (std::size_t idx = 0; idx < num_globals; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5479 [ # # # # : 0 : Module::Block().list.push_back(Module::Builder().makeGlobalSet(names_[idx], vectors[idx].expr()));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5480 : 0 : }
5481 : :
5482 : : /** Retrieve value. */
5483 : 1636 : operator PrimitiveExpr<T, L>() const requires (num_globals == 1) {
5484 [ + - # # : 1636 : return PrimitiveExpr<T, L>(Module::Builder().makeGlobalGet(names_[0], type_));
# # # # +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5485 : 0 : }
5486 : : /** Retrieve value. */
5487 : 0 : operator PrimitiveExpr<T, L>() const requires (num_globals > 1) {
5488 : : static_assert(num_globals == PrimitiveExpr<T, L>::num_vectors);
5489 : 0 : std::array<typename PrimitiveExpr<T, L>::vector_type, num_globals> vectors;
5490 [ # # # # : 0 : for (std::size_t idx = 0; idx < num_globals; ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5491 [ # # # # : 0 : vectors[idx] = typename PrimitiveExpr<T, L>::vector_type(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5492 [ # # # # : 0 : Module::Builder().makeGlobalGet(names_[idx], type_)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5493 : : );
5494 [ # # # # : 0 : return PrimitiveExpr<T, L>(std::move(vectors));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5495 : 0 : }
5496 : : };
5497 : :
5498 : : }
5499 : :
5500 : : template<typename T, VariableKind Kind, bool CanBeNull, std::size_t L>
5501 : : requires (not (dsl_pointer_to_primitive<T> and CanBeNull)) and // pointers cannot be NULL
5502 : : (not (Kind == VariableKind::Global and CanBeNull)) and // globals cannot be NULL
5503 : : requires { typename std::conditional_t<CanBeNull, Expr<T, L>, PrimitiveExpr<T, L>>; }
5504 : : struct Variable<T, Kind, CanBeNull, L>
5505 : : {
5506 : : using type = T;
5507 : : static constexpr std::size_t num_simd_lanes = L;
5508 : : template<typename X>
5509 : : using dependent_expr_t = conditional_one_t<CanBeNull, expr_t, primitive_expr_t, X>;
5510 : : using dependent_expr_type = dependent_expr_t<PrimitiveExpr<T, L>>;
5511 : :
5512 : : private:
5513 : : ///> the type of storage for this `Variable`
5514 : : using storage_type = detail::variable_storage<T, Kind, CanBeNull, L>;
5515 : : ///> storage of this `Variable`
5516 : : storage_type storage_;
5517 : : #ifdef M_ENABLE_SANITY_FIELDS
5518 : : ///> flag to insist that this `Variable` at least used once
5519 : : mutable bool used_ = false;
5520 : : #define REGISTER_USE(VAR) (VAR).used_ = true
5521 : : #else
5522 : : #define REGISTER_USE(VAR)
5523 : : #endif
5524 : :
5525 : : public:
5526 : 0 : friend void swap(Variable &first, Variable &second) {
5527 : : using std::swap;
5528 : 0 : swap(first.storage_, second.storage_);
5529 : : #ifdef M_ENABLE_SANITY_FIELDS
5530 : : swap(first.used_, second.used_);
5531 : : #endif
5532 : 0 : }
5533 : :
5534 : : /** Default-constructs a new `Variable`. */
5535 : 5518 : Variable() = default;
5536 : :
5537 : : Variable(const Variable&) = delete;
5538 : 0 : Variable(Variable &&other)
5539 : 0 : : storage_(std::forward<storage_type>(other.storage_))
5540 : : #ifdef M_ENABLE_SANITY_FIELDS
5541 : : , used_(other.used_)
5542 : : #endif
5543 : : {
5544 : : REGISTER_USE(other);
5545 : 0 : }
5546 : :
5547 [ # # ]: 0 : Variable & operator=(const Variable &other) { operator=(other.val()); return *this; }
5548 [ # # # # : 0 : Variable & operator=(Variable &&other) { operator=(other.val()); return *this; }
# # # # ]
5549 : :
5550 : 7859 : ~Variable() {
5551 : : #ifdef M_ENABLE_SANITY_FIELDS
5552 : : M_insist(used_, "variable must be used at least once");
5553 : : #endif
5554 : 7859 : }
5555 : :
5556 : : /** Constructs a new `Variable` and initializes it with \p value. */
5557 : : template<typename... Us>
5558 : : requires requires (Us&&... us) { storage_type(std::forward<Us>(us)...); }
5559 [ + - - + ]: 1694 : explicit Variable(Us&&... value) : storage_(std::forward<Us>(value)...) { }
5560 : :
5561 : : protected:
5562 : : /** Constructs a `Variable` instance from an already allocated local with the given index \p idx. Used by
5563 : : * `Parameter` to create `Variable` instances for function parameters. */
5564 : 647 : Variable(::wasm::Index idx, tag<int> tag)
5565 : : requires (Kind == VariableKind::Param)
5566 : 647 : : storage_(idx, tag)
5567 : 647 : { }
5568 : :
5569 : : public:
5570 : : /** Check whether this `Variable` can be assigned to `NULL`, i.e. it has a NULL bit to store this information.
5571 : : * This is a compile-time information. */
5572 : : constexpr bool has_null_bit() const { return CanBeNull; }
5573 : : /** Check whether the value of this `Variable` can be `NULL`. This is a runtime-time information. */
5574 : : bool can_be_null() const {
5575 : : if constexpr (CanBeNull)
5576 : : return dependent_expr_type(*this).can_be_null();
5577 : : else
5578 : : return false;
5579 : : }
5580 : :
5581 : : /** Obtain a `Variable<T, L>`s value as a `PrimitiveExpr<T, L>` or `Expr<T, L>`, depending on `CanBeNull`.
5582 : : * Although a `Variable`'s value can also be obtained through implicit conversion (see below), some C/C++
5583 : : * constructs fail to do so (e.g. arguments to calls) and it is therefore more convenient to call `val()`. */
5584 : 353 : dependent_expr_type val() const { REGISTER_USE(*this); return dependent_expr_type(storage_); }
5585 : :
5586 : : /** Obtain a `Variable<T, L>`s value as a `PrimitiveExpr<T, L>` or `Expr<T, L>`, depending on `CanBeNull`. This
5587 : : * implicit conversion enables using a `Variable` much like a `PrimitiveExpr` or `Expr`, respectively. */
5588 : 7646 : operator dependent_expr_type() const { REGISTER_USE(*this); return dependent_expr_type(storage_); }
5589 : :
5590 : : template<typename U>
5591 : : requires requires (dependent_expr_type v) { dependent_expr_t<U>(v); }
5592 : : operator dependent_expr_t<U>() const { return dependent_expr_t<U>(dependent_expr_type(*this)); }
5593 : :
5594 : : template<typename To, std::size_t ToL = L>
5595 : : requires requires (dependent_expr_type v) { v.template to<To, ToL>(); }
5596 [ + - + - : 456 : dependent_expr_t<PrimitiveExpr<To, ToL>> to() const { return dependent_expr_type(*this).template to<To, ToL>(); }
+ - + - +
- + - + -
+ - # # #
# + - +
- ]
5597 : :
5598 : : template<typename To, std::size_t ToL = L>
5599 : : requires requires (dependent_expr_type v) { v.template reinterpret<To, ToL>(); }
5600 : : dependent_expr_t<PrimitiveExpr<To, ToL>> reinterpret() const {
5601 : : return dependent_expr_type(*this).template reinterpret<To, ToL>();
5602 : : }
5603 : :
5604 : : template<std::size_t ToL>
5605 : : requires requires (dependent_expr_type v) { v.template broadcast<ToL>(); }
5606 : 0 : dependent_expr_t<PrimitiveExpr<T, ToL>> broadcast() const {
5607 [ # # # # ]: 0 : return dependent_expr_type(*this).template broadcast<ToL>();
5608 : 0 : }
5609 : :
5610 : : template<typename... Us>
5611 : : requires requires (Us&&... us) { storage_.init(std::forward<Us>(us)...); }
5612 : 1654 : void init(Us&&... init) { storage_.init(std::forward<Us>(init)...); }
5613 : :
5614 : : template<typename U>
5615 : : requires requires (U &&u) { storage_ = std::forward<U>(u); }
5616 : 2264 : Variable & operator=(U &&value) { storage_ = std::forward<U>(value); return *this; }
5617 : :
5618 : 0 : void set_true()
5619 : : requires requires (storage_type s) { { s.set_true() } -> std::same_as<void>; }
5620 : : {
5621 : 0 : storage_.set_true();
5622 : 0 : }
5623 : :
5624 : 0 : void set_false()
5625 : : requires requires (storage_type s) { { s.set_false() } -> std::same_as<void>; }
5626 : : {
5627 : 0 : storage_.set_false();
5628 : 0 : }
5629 : :
5630 : : void set_null()
5631 : : requires requires (storage_type s) { { s.set_null() } -> std::same_as<void>; }
5632 : : {
5633 : : storage_.set_null();
5634 : : }
5635 : :
5636 : :
5637 : : /*------------------------------------------------------------------------------------------------------------------
5638 : : * Forward operators on Variable<T, L>
5639 : : *----------------------------------------------------------------------------------------------------------------*/
5640 : :
5641 : : /*----- Unary operators ------------------------------------------------------------------------------------------*/
5642 : : #define UNARY(OP) \
5643 : : auto OP() const requires requires (dependent_expr_type e) { e.OP(); } { return dependent_expr_type(*this).OP(); }
5644 : :
5645 [ + - + - : 52 : UNARY_LIST(UNARY)
+ - + - +
- + - + -
# # + - +
- + - + -
+ - + - +
- ]
5646 : : UNARY(add_pairwise) // from PrimitiveExpr and Expr
5647 : : UNARY(bitmask) // from PrimitiveExpr and Expr
5648 : : UNARY(any_true) // from PrimitiveExpr and Expr
5649 [ # # # # : 0 : UNARY(all_true) // from PrimitiveExpr and Expr
# # # # #
# ]
5650 : : UNARY(hash) // from PrimitiveExpr and Expr
5651 [ + - + - : 356 : UNARY(operator *) // from PrimitiveExpr for pointers
+ - + - #
# + - ]
5652 : : UNARY(operator ->) // from PrimitiveExpr for pointers
5653 [ + - # # ]: 16 : UNARY(is_nullptr) // from PrimitiveExpr for pointers
5654 [ # # + - : 58 : UNARY(is_null) // from Expr
+ - # # #
# # # +
- ]
5655 [ + - # # : 8 : UNARY(not_null) // from Expr
# # # # #
# + - #
# ]
5656 [ + - ]: 6 : UNARY(is_true_and_not_null) // from Expr
5657 [ + - ]: 6 : UNARY(is_false_and_not_null) // from Expr
5658 : : #undef UNARY
5659 : :
5660 : : /*----- Assignment operators -------------------------------------------------------------------------------------*/
5661 : : #define ASSIGNOP_LIST(X) \
5662 : : X(+) \
5663 : : X(-) \
5664 : : X(*) \
5665 : : X(/) \
5666 : : X(%) \
5667 : : X(&) \
5668 : : X(|) \
5669 : : X(^) \
5670 : : X(<<) \
5671 : : X(>>)
5672 : :
5673 : : #define ASSIGNOP(SYMBOL) \
5674 : : template<typename U> \
5675 : : requires requires { typename dependent_expr_t<U>; } and \
5676 : : requires (U &&u) { dependent_expr_t<U>(std::forward<U>(u)); } and \
5677 : : requires (dependent_expr_type var_value, dependent_expr_t<U> other_value) \
5678 : : { var_value SYMBOL other_value; } and \
5679 : : requires (Variable var, \
5680 : : decltype(std::declval<dependent_expr_type>() SYMBOL std::declval<dependent_expr_t<U>>()) value) \
5681 : : { var = value; } \
5682 : : Variable & operator SYMBOL##= (U &&value) { \
5683 : : dependent_expr_t<U> _value(std::forward<U>(value)); \
5684 : : this->operator=(dependent_expr_type(*this) SYMBOL _value); \
5685 : : return *this; \
5686 : : }
5687 [ + - + - : 894 : ASSIGNOP_LIST(ASSIGNOP)
+ - - + +
- + - + -
- + + - +
- + - - +
+ - + - +
- - + # #
# # # # #
# + - + -
+ - - + +
- + - + -
- + + - +
- + - - +
# # # # #
# # # + -
+ - + - -
+ # # # #
# # # # #
# # # # #
# # + - +
- + - - +
# # # # #
# # # # #
# # # # #
# + - + -
+ - - + #
# # # # #
# # # # #
# # # # #
+ - + - +
- - + # #
# # # # #
# + - + -
+ - - + +
- + - + -
- + + - +
- + - - +
# # # # #
# # # + -
+ - + - -
+ # # # #
# # # # #
# # # # #
# # + - +
- + - - +
# # # # #
# # # # #
# # # # #
# + - + -
+ - - + #
# # # # #
# # # # #
# # # # #
+ - + - +
- - + # #
# # # # #
# # # # #
# # # # +
- + - + -
- + # # #
# # # # #
# # # # #
# # # + -
+ - + - -
+ # # # #
# # # # #
# # # # #
# # + - +
- + - - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5688 : : #undef ASSIGNOP
5689 : :
5690 : : /*----- Modifications --------------------------------------------------------------------------------------------*/
5691 : : /** Extracts the \tparam M -th value of `this`. */
5692 : : template<std::size_t M>
5693 : 0 : auto extract() const requires requires (dependent_expr_type e) { e.template extract<M>(); } {
5694 [ # # # # : 0 : return dependent_expr_type(*this).template extract<M>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5695 : 0 : }
5696 : :
5697 : : /** Replaces the \tparam M -th value of `this` with \p value. */
5698 : : template<std::size_t M, typename U>
5699 : : Variable & replace(U &&value)
5700 : : requires requires (dependent_expr_type e) { e.template replace<M>(std::forward<U>(value)); } {
5701 : : this->operator=(dependent_expr_type(*this).template replace<M>(std::forward<U>(value)));
5702 : : return *this;
5703 : : }
5704 : :
5705 : : /** Selects lanes of `this` in byte granularity depending on the indices specified by \p indices. Indices `i` in
5706 : : * the range [0, 15] select the `i`-th` lane, indices outside of this range result in the value 0. */
5707 : : auto swizzle_bytes(PrimitiveExpr<uint8_t, 16> indices) const
5708 : : requires requires (dependent_expr_type e) { e.swizzle_bytes(indices); } {
5709 : : return dependent_expr_type(*this).swizzle_bytes(indices);
5710 : : }
5711 : :
5712 : : /** Selects lanes of `this` in lane granularity depending on the indices specified by \p indices. Indices `i` in
5713 : : * the range [0, L) select the `i`-th` lane, indices outside of this range result in the value 0. */
5714 : : template<std::size_t M>
5715 : : auto swizzle_lanes(const std::array<uint8_t, M> &indices) const
5716 : : requires requires (dependent_expr_type e) { e.swizzle_lanes(indices); } {
5717 : : return dependent_expr_type(*this).swizzle_lanes(indices);
5718 : : }
5719 : :
5720 : : #undef REGISTER_USE
5721 : : };
5722 : :
5723 : : /*----- Overload forwarded binary operators for pointer advancing on PrimitiveExpr<T*, L> ----------------------------*/
5724 : : template<dsl_pointer_to_primitive T, VariableKind Kind, bool CanBeNull, std::size_t L>
5725 : : requires requires (const Variable<T, Kind, CanBeNull, L> &var, typename PrimitiveExpr<T, L>::offset_t delta)
5726 : : { var.val().operator+(delta); }
5727 : 54 : auto operator+(const Variable<T, Kind, CanBeNull, L> &var, typename PrimitiveExpr<T, L>::offset_t delta)
5728 : : {
5729 [ # # + - ]: 54 : return var.val().operator+(delta);
5730 : 0 : }
5731 : :
5732 : : template<dsl_pointer_to_primitive T, VariableKind Kind, bool CanBeNull, std::size_t L>
5733 : : requires requires (const Variable<T, Kind, CanBeNull, L> &var, typename PrimitiveExpr<T, L>::offset_t delta)
5734 : : { var.val().operator-(delta); }
5735 : 0 : auto operator-(const Variable<T, Kind, CanBeNull, L> &var, typename PrimitiveExpr<T, L>::offset_t delta)
5736 : : {
5737 [ # # ]: 0 : return var.val().operator-(delta);
5738 : 0 : }
5739 : :
5740 : : namespace detail {
5741 : :
5742 : : /** Deduces a suitable specialization of `Variable` for the given type \tparam T. */
5743 : : template<typename T>
5744 : : struct var_helper;
5745 : :
5746 : : template<typename T, std::size_t L>
5747 : : struct var_helper<PrimitiveExpr<T, L>>
5748 : : { using type = Variable<T, VariableKind::Local, /* CanBeNull= */ false, L>; };
5749 : :
5750 : : template<typename T, std::size_t L>
5751 : : struct var_helper<Expr<T, L>>
5752 : : { using type = Variable<T, VariableKind::Local, /* CanBeNull= */ true, L>; };
5753 : :
5754 : : /** Deduces a suitable specialization of `Variable` *that can be NULL* for the given type \tparam T. */
5755 : : template<typename T>
5756 : : struct _var_helper;
5757 : :
5758 : : template<typename T, std::size_t L>
5759 : : struct _var_helper<PrimitiveExpr<T, L>>
5760 : : { using type = Variable<T, VariableKind::Local, /* CanBeNull= */ true, L>; };
5761 : :
5762 : : template<typename T, std::size_t L>
5763 : : struct _var_helper<Expr<T, L>>
5764 : : { using type = Variable<T, VariableKind::Local, /* CanBeNull= */ true, L>; };
5765 : :
5766 : : /** Deduces a suitable specialization of `Variable` for global variables of the given type \tparam T. */
5767 : : template<typename T>
5768 : : struct global_helper;
5769 : :
5770 : : template<typename T, std::size_t L>
5771 : : struct global_helper<PrimitiveExpr<T, L>>
5772 : : { using type = Variable<T, VariableKind::Global, /* CanBeNull= */ false, L>; };
5773 : :
5774 : : }
5775 : :
5776 : : /** Local variable. Can be `NULL` if \tparam T can be `NULL`. */
5777 : : template<typename T>
5778 : : requires requires { typename detail::var_helper<T>::type; }
5779 : : using Var = typename detail::var_helper<T>::type;
5780 : :
5781 : : /** Local variable that *can always* be `NULL`. */
5782 : : template<typename T>
5783 : : requires requires { typename detail::_var_helper<T>::type; }
5784 : : using _Var = typename detail::_var_helper<T>::type;
5785 : :
5786 : : /** Global variable. Cannot be `NULL`. */
5787 : : template<typename T>
5788 : : requires requires { typename detail::global_helper<T>::type; }
5789 : : using Global = typename detail::global_helper<T>::type;
5790 : :
5791 : :
5792 : : /*======================================================================================================================
5793 : : * Parameter
5794 : : *====================================================================================================================*/
5795 : :
5796 : : /** A type to access function parameters. Function parameters are like local variables, but they need not be explicitly
5797 : : * allocated on the stack but are implicitly allocated by the function's signature. Parameters are indexed in the order
5798 : : * they occur in the function signature. */
5799 : : template<typename T, std::size_t L>
5800 : : requires (L * sizeof(T) <= 16) // parameter must fit in a single Wasm local
5801 : : struct Parameter<T, L> : Variable<T, VariableKind::Param, /* CanBeNull= */ false, L>
5802 : : {
5803 : : template<typename>
5804 : : friend struct Function; // to enable `Function` to create `Parameter` instances through private c'tor
5805 : :
5806 : : using base_type = Variable<T, VariableKind::Param, /* CanBeNull= */ false, L>;
5807 : : using base_type::operator=;
5808 : : using dependent_expr_type = typename base_type::dependent_expr_type;
5809 : : using base_type::operator dependent_expr_type;
5810 : :
5811 : : private:
5812 : : /** Create a `Parameter<T, L>` for the existing parameter local of given `index`. Parameters can only be created by
5813 : : * `Function::parameter<I>()`. */
5814 : 647 : Parameter(unsigned index)
5815 : 647 : : base_type(::wasm::Index(index), tag<int>{})
5816 : : {
5817 [ + - + - : 647 : ::wasm::Function &fn = Module::Function();
+ - + - +
- + - # #
# # + - +
- + - + -
# # + - +
- ]
5818 [ + - + - : 647 : M_insist(index < fn.getNumLocals(), "index out of bounds");
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - #
# # # + -
+ - + - +
- ]
5819 [ + - + - : 647 : M_insist(fn.isParam(index), "not a parameter");
+ - + - +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - #
# # # + -
+ - + - +
- ]
5820 [ + - + - : 647 : M_insist(fn.getLocalType(index) == (wasm_type<T, L>()), "type mismatch");
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # #
# # # # #
# # # # +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - # #
# # # # +
- + - + -
+ - + - +
- ]
5821 : 647 : }
5822 : : };
5823 : :
5824 : :
5825 : : /*======================================================================================================================
5826 : : * Pointer and References
5827 : : *====================================================================================================================*/
5828 : :
5829 : : namespace detail {
5830 : :
5831 : : template<dsl_primitive T, std::size_t L, bool IsConst>
5832 : : struct the_reference
5833 : : {
5834 : : friend struct PrimitiveExpr<T*, L>; // to construct a reference to the pointed-to memory
5835 : : friend struct Variable<T*, VariableKind::Local, false, L>; // to construct a reference to the pointed-to memory
5836 : : friend struct Variable<T*, VariableKind::Global, false, L>; // to construct a reference to the pointed-to memory
5837 : : friend struct Variable<T*, VariableKind::Param, false, L>; // to construct a reference to the pointed-to memory
5838 : :
5839 : : static constexpr bool is_const = IsConst;
5840 : :
5841 : : private:
5842 : : PrimitiveExpr<T*, L> ptr_;
5843 : :
5844 : : private:
5845 : 1254 : explicit the_reference(PrimitiveExpr<T*, L> ptr)
5846 : 1254 : : ptr_(ptr)
5847 : : {
5848 [ + - + - : 1254 : M_insist(bool(ptr_), "must not be moved or discarded");
+ - + - +
- + - + -
+ - # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5849 : 1254 : }
5850 : :
5851 : : public:
5852 : : template<typename U>
5853 : : requires (not is_const) and requires (U &&u) { PrimitiveExpr<T, L>(primitive_expr_t<U>(std::forward<U>(u))); }
5854 : 676 : void operator=(U &&_value) {
5855 : 676 : PrimitiveExpr<T, L> value(primitive_expr_t<U>(std::forward<U>(_value)));
5856 [ + - - + : 676 : ptr_.store(value);
+ - - + +
- - + + -
- + + - -
+ + - - +
# # # # +
- - + + -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5857 : 676 : }
5858 : :
5859 : : ///> implicit loading of the referenced value
5860 : 578 : operator PrimitiveExpr<T, L>() { return ptr_.load(); }
5861 : :
5862 : : #define ASSIGNOP(SYMBOL) \
5863 : : template<typename U> \
5864 : : requires requires (the_reference ref, U &&u) { ref SYMBOL std::forward<U>(u); } and \
5865 : : requires (the_reference ref, decltype(ref SYMBOL std::declval<U>()) value) \
5866 : : { ref = value; } \
5867 : : void operator SYMBOL##= (U &&value) { \
5868 : : this->operator=(the_reference(ptr_.clone()) SYMBOL std::forward<U>(value)); \
5869 : : }
5870 [ + - + - : 8 : ASSIGNOP_LIST(ASSIGNOP)
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5871 : : #undef ASSIGNOP
5872 : : };
5873 : :
5874 : : }
5875 : :
5876 : :
5877 : : /*======================================================================================================================
5878 : : * LocalBitmap, LocalBitvector, Bit, and LocalBit
5879 : : *====================================================================================================================*/
5880 : :
5881 : : struct LocalBitmap
5882 : : {
5883 : : friend struct Module;
5884 : :
5885 : : Var<U64x1> u64;
5886 : 184 : uint64_t bitmask = uint64_t(-1UL);
5887 : :
5888 : : private:
5889 : 368 : LocalBitmap() = default;
5890 : : LocalBitmap(const LocalBitmap&) = delete;
5891 : : };
5892 : :
5893 : : struct LocalBitvector
5894 : : {
5895 : : friend struct Module;
5896 : :
5897 : : Var<U8x16> u8x16;
5898 : : ///> entry at index `i` is a bitmask for the 16 lanes using the constant bit offset `i`
5899 : : std::array<uint16_t, 8> bitmask_per_offset;
5900 : :
5901 : : private:
5902 : 16 : LocalBitvector()
5903 : : {
5904 [ + - ]: 16 : bitmask_per_offset.fill(uint16_t(-1U));
5905 : 16 : }
5906 : : LocalBitvector(const LocalBitvector&) = delete;
5907 : : };
5908 : :
5909 : : struct Bit
5910 : : {
5911 : 1256 : virtual ~Bit() { }
5912 : : };
5913 : :
5914 : : namespace detail {
5915 : :
5916 : : /** Helper class to select appropriate storage for a `LocalBit`. */
5917 : : template<std::size_t L>
5918 : : requires (L <= 16)
5919 : : class local_bit_storage
5920 : : {
5921 : : friend struct LocalBit<L>; // to be usable by the respective LocalBit
5922 : :
5923 : 32 : friend void swap(local_bit_storage &first, local_bit_storage &second) {
5924 : : using std::swap;
5925 : 32 : swap(first.bitvector_, second.bitvector_);
5926 : 32 : swap(first.bit_offset_, second.bit_offset_);
5927 : 32 : swap(first.starting_lane_, second.starting_lane_);
5928 : 32 : }
5929 : :
5930 : 32 : LocalBitvector *bitvector_ = nullptr; ///< the bitvector in which the *multiple* bits are contained
5931 : : uint8_t bit_offset_; ///< the offset of each bit in every lane
5932 : : uint8_t starting_lane_; ///< the lane index at which the L bits start
5933 : :
5934 : 64 : local_bit_storage() = default;
5935 : : /** Creates multiple bits with storage allocated in \p bitvector. */
5936 : 32 : local_bit_storage(LocalBitvector &bitvector, uint8_t bit_offset, uint8_t starting_lane)
5937 : 32 : : bitvector_(&bitvector)
5938 : 32 : , bit_offset_(bit_offset)
5939 : 32 : , starting_lane_(starting_lane)
5940 : : {
5941 : 32 : M_insist(bit_offset_ < 8, "offset out of bounds");
5942 : 32 : M_insist(starting_lane_ + L <= 16, "starting lane out of bounds");
5943 : 32 : }
5944 : : };
5945 : :
5946 : : /** Specialization for a *single* bit. */
5947 : : template<>
5948 : : class local_bit_storage<1>
5949 : : {
5950 : : friend struct LocalBit<1>; // to be usable by the respective LocalBit
5951 : :
5952 : 596 : friend void swap(local_bit_storage &first, local_bit_storage &second) {
5953 : : using std::swap;
5954 : 596 : swap(first.bitmap_, second.bitmap_);
5955 : 596 : swap(first.bit_offset_, second.bit_offset_);
5956 : 596 : }
5957 : :
5958 : 596 : LocalBitmap *bitmap_ = nullptr; ///< the bitmap in which the *single* bit is contained
5959 : : uint8_t bit_offset_; ///< the offset of the *single* bit
5960 : :
5961 : 1192 : local_bit_storage() = default;
5962 : : /** Creates a single bit with storage allocated in \p bitmap. */
5963 : 596 : local_bit_storage(LocalBitmap &bitmap, uint8_t bit_offset) : bitmap_(&bitmap), bit_offset_(bit_offset)
5964 : : {
5965 : 596 : M_insist(bit_offset_ < CHAR_BIT * sizeof(uint64_t), "offset out of bounds");
5966 : 596 : }
5967 : : };
5968 : :
5969 : : }
5970 : :
5971 : : /**
5972 : : * A scalar bit or a vector of bits that is managed by the current function's stack.
5973 : : *
5974 : : * 0 ⇔ false ⇔ NOT NULL
5975 : : * 1 ⇔ true ⇔ NULL
5976 : : */
5977 : : template<std::size_t L>
5978 : : requires (L > 0) and (L <= 16)
5979 : : struct LocalBit<L> : Bit
5980 : : {
5981 : 628 : friend void swap(LocalBit &first, LocalBit &second) {
5982 : : using std::swap;
5983 : 628 : swap(first.storage_, second.storage_);
5984 : 628 : }
5985 : :
5986 : : friend struct Module; // to construct LocalBit
5987 : :
5988 : : private:
5989 : : using storage_type = detail::local_bit_storage<L>;
5990 : : storage_type storage_;
5991 : :
5992 : 628 : LocalBit() = default;
5993 : :
5994 : : template<typename... Us>
5995 : : requires requires (Us&&... us) { storage_type(std::forward<Us>(us)...); }
5996 [ + - + - : 628 : LocalBit(Us&&... us) : storage_(std::forward<Us>(us)...) { }
# # + - +
- ]
5997 : :
5998 : : public:
5999 : : LocalBit(const LocalBit&) = delete;
6000 [ + - + - : 628 : LocalBit(LocalBit &&other) : LocalBit() { swap(*this, other); }
# # + - +
- ]
6001 : :
6002 : : /** Must not be defined out-of-line due to a LLVM clang issue, see https://bugs.llvm.org/show_bug.cgi?id=46979#c1. */
6003 : 1256 : ~LocalBit() {
6004 : : if constexpr (L == 1) {
6005 [ + + ]: 1192 : if (storage_.bitmap_) {
6006 [ + - + - ]: 596 : M_insist((storage_.bitmap_->bitmask bitand mask()) == 0, "bit must still be allocated");
6007 : :
6008 [ - + ]: 596 : if (storage_.bitmap_->bitmask == 0) // empty bitmap
6009 [ # # # # ]: 0 : Module::Get().local_bitmaps_stack_.back().emplace_back(storage_.bitmap_); // make discoverable again
6010 : :
6011 [ + - ]: 596 : storage_.bitmap_->bitmask |= mask(); // deallocate bit
6012 : 596 : }
6013 : : } else {
6014 [ + + # # : 64 : if (storage_.bitvector_) {
+ + + + ]
6015 : 32 : constexpr uint16_t MASK = (1U << L) - 1U;
6016 [ + - + - : 32 : M_insist((storage_.bitvector_->bitmask_per_offset[offset()] bitand (MASK << starting_lane())) == 0,
+ - # # #
# # # + -
+ - + - +
- + - +
- ]
6017 : : "bits must still be allocated");
6018 : :
6019 : 32 : const auto &bitmasks = storage_.bitvector_->bitmask_per_offset;
6020 : 92 : auto pred = [](auto bitmask){ return bitmask == 0; };
6021 [ + - - + : 32 : if (std::all_of(bitmasks.cbegin(), bitmasks.cend(), pred)) // empty bitvector
# # # # +
- - + + -
- + ]
6022 [ # # # # : 0 : Module::Get().local_bitvectors_stack_.back().emplace_back(storage_.bitvector_); // make discoverable again
# # # # #
# # # # #
# # ]
6023 : :
6024 [ + - + - : 32 : storage_.bitvector_->bitmask_per_offset[offset()] |= MASK << starting_lane(); // deallocate bits
# # # # +
- + - + -
+ - ]
6025 : 32 : }
6026 : : }
6027 : 1256 : }
6028 : :
6029 : : LocalBit & operator=(LocalBit &&other) { swap(*this, other); return *this; }
6030 : :
6031 : : private:
6032 : : ///> Returns the offset of the bits.
6033 : 3212 : std::conditional_t<L == 1, uint64_t, uint8_t> offset() const { return storage_.bit_offset_; }
6034 : : ///> Returns a mask with a single bit set at offset `offset()`.
6035 : 2428 : uint64_t mask() const requires (L == 1) { return 1UL << offset(); }
6036 : : ///> Returns a mask with a single bit set per lane at offset `offset()`.
6037 : 32 : U8x16 mask() const requires (L > 1) { return U8x16(1U << offset()); }
6038 : : ///> Returns a mask with a single bit unset per lane at offset `offset()`.
6039 : 32 : U8x16 mask_inverted() const requires (L > 1) { return U8x16(~(1U << offset())); }
6040 : : ///> Returns the lane index at which the L bits start.
6041 : 128 : uint8_t starting_lane() const requires (L > 1) { return storage_.starting_lane_; }
6042 : :
6043 : : public:
6044 : : /** Returns the boolean expression that evaluates to `true` if the respective bits are set, `false` otherwise. */
6045 : 644 : PrimitiveExpr<bool, L> is_set() const {
6046 : : if constexpr (L == 1) {
6047 [ + - ]: 612 : return (storage_.bitmap_->u64 bitand mask()).template to<bool>();
6048 : : } else {
6049 : : if constexpr (L == 16) { // all lanes used
6050 : 12 : M_insist(starting_lane() == 0);
6051 [ + - - + ]: 12 : return (storage_.bitvector_->u8x16 bitand mask()).template to<bool>();
6052 [ # # - + : 20 : } else if (starting_lane()) { // swizzle lanes to correct starting point
- + ]
6053 : : std::array<uint8_t, L> indices;
6054 : 0 : std::iota(indices.begin(), indices.end(), starting_lane()); // fill with [starting_lane(), starting_lane() + L)
6055 [ # # # # : 0 : return (storage_.bitvector_->u8x16 bitand mask()).swizzle_bytes(indices).template to<bool>();
# # # # #
# # # # #
# # # # ]
6056 : : } else { // no swizzling needed, but explicitly convert s.t. only first L lanes are used
6057 [ # # # # : 20 : return PrimitiveExpr<bool, L>((storage_.bitvector_->u8x16 bitand mask()).template to<bool>().move());
# # # # +
- + - + -
- + + - +
- + - -
+ ]
6058 : : }
6059 : : }
6060 : 20 : }
6061 : :
6062 : : /** Sets all bits. */
6063 : 0 : void set() {
6064 : : if constexpr (L == 1)
6065 : 0 : storage_.bitmap_->u64 |= mask();
6066 : : else
6067 : : storage_.bitvector_->u8x16 |= mask();
6068 : 0 : }
6069 : : /** Clears all bits. */
6070 : 0 : void clear() {
6071 : : if constexpr (L == 1)
6072 : 0 : storage_.bitmap_->u64 &= ~mask();
6073 : : else
6074 [ # # # # : 0 : storage_.bitvector_->u8x16 &= mask_inverted();
# # # # ]
6075 : 0 : }
6076 : :
6077 : : /** Sets these bits to the boolean values of \p value. */
6078 : 656 : void set(PrimitiveExpr<bool, L> value) {
6079 : : if constexpr (L == 1) {
6080 [ - + ]: 624 : storage_.bitmap_->u64 =
6081 [ + - + - : 624 : (storage_.bitmap_->u64 bitand ~mask()) bitor (value.template to<uint64_t>() << offset());
+ - + - ]
6082 : : } else {
6083 : : if constexpr (L == 16) { // all lanes used
6084 : 12 : M_insist(starting_lane() == 0);
6085 [ + - + - : 24 : storage_.bitvector_->u8x16 = (storage_.bitvector_->u8x16 bitand mask_inverted()) bitor
- + ]
6086 [ + - + - ]: 12 : (value.template to<uint8_t>() << offset());
6087 [ # # - + : 20 : } else if (starting_lane()) { // swizzle lanes to correct starting point, other lanes are 0
- + ]
6088 : : std::array<uint8_t, 16> indices;
6089 : 0 : auto it = std::fill_n(indices.begin(), starting_lane(), L); // fill range [0, starting_lane()) with L
6090 : 0 : std::iota(it, it + L, 0); // fill range [starting_lane(), starting_lane() + L) with [0, L)
6091 : 0 : std::fill(it + L, indices.end(), L); // fill range [starting_lane() + L, 16) with L
6092 [ # # # # : 0 : storage_.bitvector_->u8x16 = (storage_.bitvector_->u8x16 bitand mask_inverted()) bitor
# # # # #
# # # # #
# # # # ]
6093 [ # # # # : 0 : (value.swizzle_bytes(indices).template to<uint8_t>() << offset());
# # # # #
# # # # #
# # # # ]
6094 : 0 : } else { // no swizzling needed, but explicitly mask s.t. last (unused) 16-L lanes are 0
6095 [ # # # # : 20 : Boolx16 value_masked((PrimitiveExpr<bool, L>(true) and value).move());
# # # # +
- + - + -
+ - + - +
- + - +
- ]
6096 [ # # # # : 40 : storage_.bitvector_->u8x16 = (storage_.bitvector_->u8x16 bitand mask_inverted()) bitor
# # # # +
- + - + -
- + + - +
- + - -
+ ]
6097 [ # # # # : 20 : (value_masked.template to<uint8_t>() << offset());
+ - + - +
- + - ]
6098 : 20 : }
6099 : : }
6100 : 656 : }
6101 : :
6102 : : /** Sets the bits of `this` to the values of the bits of \p other. Cleverly computes required shift width at
6103 : : * compile time to use only a single shift operation. */
6104 : : LocalBit & operator=(const LocalBit &other) {
6105 : : if constexpr (L == 1) {
6106 : : auto other_bit = other.storage_.bitmap_->u64 bitand other.mask();
6107 : : Var<U64x1> this_bit;
6108 : :
6109 : : if (this->offset() > other.offset()) {
6110 : : const auto shift_width = this->offset() - other.offset();
6111 : : this_bit = other_bit << shift_width;
6112 : : } else if (other.offset() > this->offset()) {
6113 : : const auto shift_width = other.offset() - this->offset();
6114 : : this_bit = other_bit >> shift_width;
6115 : : } else {
6116 : : this_bit = other_bit;
6117 : : }
6118 : :
6119 : : this->storage_.bitmap_->u64 =
6120 : : (this->storage_.bitmap_->u64 bitand ~this->mask()) bitor this_bit; // clear, then set bit
6121 : :
6122 : : return *this;
6123 : : } else {
6124 : : auto other_bits = other.storage_.bitvector_->u8x16 bitand other.mask();
6125 : : Var<U8x16> this_bits;
6126 : :
6127 : : if (this->starting_lane() != other.starting_lane()) {
6128 : : std::array<uint8_t, 16> indices;
6129 : : auto it = std::fill_n(indices.begin(), starting_lane(), L); // fill range [0, starting_lane()) with L
6130 : : std::iota(it, it + L, 0); // fill range [starting_lane(), starting_lane() + L) with [0, L)
6131 : : std::fill(it + L, indices.end(), L); // fill range [starting_lane() + L, 16) with L
6132 : : this_bits = other_bits.swizzle_bytes(indices);
6133 : : } else {
6134 : : this_bits = other_bits;
6135 : : }
6136 : :
6137 : : if (this->offset() > other.offset()) {
6138 : : const auto shift_width = this->offset() - other.offset();
6139 : : this_bits = other_bits << shift_width;
6140 : : } else if (other.offset() > this->offset()) {
6141 : : const auto shift_width = other.offset() - this->offset();
6142 : : this_bits = other_bits >> shift_width;
6143 : : } else {
6144 : : this_bits = other_bits;
6145 : : }
6146 : :
6147 : : this->storage_.bitvector_->u8x16 =
6148 : : (this->storage_.bitvector_->u8x16 bitand this->mask_inverted()) bitor this_bits; // clear, then set bit
6149 : :
6150 : : return *this;
6151 : : }
6152 : : }
6153 : :
6154 : : /** Converts this `LocalBit` to `PrimitiveExpr<bool, L>`, which is `true` iff this `LocalBit` is set. */
6155 : : operator PrimitiveExpr<bool, L>() const { return is_set(); }
6156 : : };
6157 : :
6158 : :
6159 : : /*======================================================================================================================
6160 : : * Control flow
6161 : : *====================================================================================================================*/
6162 : :
6163 : : /*----- Return unsafe, i.e. without static type checking -------------------------------------------------------------*/
6164 : :
6165 : : inline void RETURN_UNSAFE() { Module::Get().emit_return(); }
6166 : :
6167 : : template<primitive_convertible T>
6168 : : inline void RETURN_UNSAFE(T &&t) { Module::Get().emit_return(primitive_expr_t<T>(std::forward<T>(t))); }
6169 : :
6170 : : template<expr_convertible T>
6171 : : requires (not primitive_convertible<T>)
6172 : : inline void RETURN_UNSAFE(T &&t) { Module::Get().emit_return(expr_t<T>(std::forward<T>(t))); }
6173 : :
6174 : : /*----- BREAK --------------------------------------------------------------------------------------------------------*/
6175 : :
6176 : 130 : inline void BREAK(std::size_t level = 1) { Module::Get().emit_break(level); }
6177 : : template<primitive_convertible C>
6178 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6179 : 180 : inline void BREAK(C &&_cond, std::size_t level = 1)
6180 : : {
6181 : 180 : PrimitiveExpr<bool, 1> cond(std::forward<C>(_cond));
6182 [ + - + - : 180 : Module::Get().emit_break(cond, level);
- + ]
6183 : 180 : }
6184 : :
6185 : : /*----- CONTINUE -----------------------------------------------------------------------------------------------------*/
6186 : :
6187 : 192 : inline void CONTINUE(std::size_t level = 1) { Module::Get().emit_continue(level); }
6188 : : template<primitive_convertible C>
6189 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6190 : 20 : inline void CONTINUE(C &&_cond, std::size_t level = 1)
6191 : : {
6192 : 20 : PrimitiveExpr<bool, 1> cond(std::forward<C>(_cond));
6193 [ + - + - : 20 : Module::Get().emit_continue(cond, level);
- + ]
6194 : 20 : }
6195 : :
6196 : : /*----- GOTO ---------------------------------------------------------------------------------------------------------*/
6197 : :
6198 : : /** Jumps to the end of \p block. */
6199 : 0 : inline void GOTO(const Block &block) { block.go_to(); }
6200 : : template<primitive_convertible C>
6201 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6202 : : /** Jumps to the end of \p block iff \p _cond is fulfilled. */
6203 : 0 : inline void GOTO(C &&_cond, const Block &block)
6204 : : {
6205 : 0 : PrimitiveExpr<bool, 1> cond(std::forward<C>(_cond));
6206 [ # # # # ]: 0 : block.go_to(cond);
6207 : 0 : }
6208 : :
6209 : : /*----- Select -------------------------------------------------------------------------------------------------------*/
6210 : :
6211 : : template<primitive_convertible C, primitive_convertible T, primitive_convertible U>
6212 : : requires have_common_type<typename primitive_expr_t<T>::type, typename primitive_expr_t<U>::type> and
6213 : : (primitive_expr_t<T>::num_simd_lanes == primitive_expr_t<U>::num_simd_lanes) and
6214 : : requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6215 : 384 : inline auto Select(C &&_cond, T &&_tru, U &&_fals)
6216 : : {
6217 : 384 : primitive_expr_t<T> tru(std::forward<T>(_tru));
6218 [ + - + - : 384 : primitive_expr_t<U> fals(std::forward<U>(_fals));
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6219 : :
6220 : : using To = common_type_t<typename decltype(tru)::type, typename decltype(fals)::type>;
6221 : 384 : constexpr std::size_t L = decltype(tru)::num_simd_lanes;
6222 : :
6223 [ + - + - : 384 : PrimitiveExpr<bool, 1> cond(std::forward<C>(_cond));
+ - + - +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6224 [ + - + - : 384 : return Module::Get().emit_select<To, L>(cond, tru.template to<To, L>(), fals.template to<To, L>());
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ + - + -
+ - + - -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
6225 : 384 : }
6226 : :
6227 : : template<primitive_convertible C, expr_convertible T, expr_convertible U>
6228 : : requires (not primitive_convertible<T> or not primitive_convertible<U>) and
6229 : : have_common_type<typename expr_t<T>::type, typename expr_t<U>::type> and
6230 : : (expr_t<T>::num_simd_lanes == expr_t<U>::num_simd_lanes) and
6231 : : requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6232 : 0 : inline auto Select(C &&_cond, T &&_tru, U &&_fals)
6233 : : {
6234 : 0 : expr_t<T> tru(std::forward<T>(_tru));
6235 [ # # # # : 0 : expr_t<U> fals(std::forward<U>(_fals));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6236 : :
6237 : : using To = common_type_t<typename decltype(tru)::type, typename decltype(fals)::type>;
6238 : 0 : constexpr std::size_t L = decltype(tru)::num_simd_lanes;
6239 : :
6240 [ # # # # : 0 : PrimitiveExpr<bool, 1> cond(std::forward<C>(_cond));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6241 [ # # # # : 0 : return Module::Get().emit_select<To, L>(cond, tru.template to<To, L>(), fals.template to<To, L>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
6242 : 0 : }
6243 : :
6244 : : template<primitive_convertible C, primitive_convertible T, primitive_convertible U>
6245 : : requires have_common_type<typename primitive_expr_t<T>::type, typename primitive_expr_t<U>::type> and
6246 : : (primitive_expr_t<T>::num_simd_lanes == primitive_expr_t<U>::num_simd_lanes) and
6247 : : (primitive_expr_t<T>::num_simd_lanes > 1) and
6248 : : requires (C &&c) { PrimitiveExpr<bool, primitive_expr_t<T>::num_simd_lanes>(std::forward<C>(c)); }
6249 : 6 : inline auto Select(C &&_cond, T &&_tru, U &&_fals)
6250 : : {
6251 : 6 : primitive_expr_t<T> tru(std::forward<T>(_tru));
6252 [ + - + - : 6 : primitive_expr_t<U> fals(std::forward<U>(_fals));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6253 : :
6254 : : using To = common_type_t<typename decltype(tru)::type, typename decltype(fals)::type>;
6255 : 6 : constexpr std::size_t L = decltype(tru)::num_simd_lanes;
6256 : :
6257 [ + - + - : 6 : PrimitiveExpr<bool, L> cond(std::forward<C>(_cond));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6258 [ + - + - : 6 : return Module::Get().emit_select<To, L>(cond, tru.template to<To, L>(), fals.template to<To, L>());
+ - + - -
+ + - + -
+ - + - -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
6259 : 6 : }
6260 : :
6261 : : template<primitive_convertible C, expr_convertible T, expr_convertible U>
6262 : : requires (not primitive_convertible<T> or not primitive_convertible<U>) and
6263 : : have_common_type<typename expr_t<T>::type, typename expr_t<U>::type> and
6264 : : (expr_t<T>::num_simd_lanes == expr_t<U>::num_simd_lanes) and (expr_t<T>::num_simd_lanes > 1) and
6265 : : requires (C &&c) { PrimitiveExpr<bool, expr_t<T>::num_simd_lanes>(std::forward<C>(c)); }
6266 : 0 : inline auto Select(C &&_cond, T &&_tru, U &&_fals)
6267 : : {
6268 : 0 : expr_t<T> tru(std::forward<T>(_tru));
6269 [ # # # # : 0 : expr_t<U> fals(std::forward<U>(_fals));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6270 : :
6271 : : using To = common_type_t<typename decltype(tru)::type, typename decltype(fals)::type>;
6272 : 0 : constexpr std::size_t L = decltype(tru)::num_simd_lanes;
6273 : :
6274 [ # # # # : 0 : PrimitiveExpr<bool, L> cond(std::forward<C>(_cond));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6275 [ # # # # : 0 : return Module::Get().emit_select<To, L>(cond, tru.template to<To, L>(), fals.template to<To, L>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
6276 : 0 : }
6277 : :
6278 : :
6279 : : /*----- Shuffle ------------------------------------------------------------------------------------------------------*/
6280 : :
6281 : : template<primitive_convertible T, primitive_convertible U, std::size_t M>
6282 : : requires have_common_type<typename primitive_expr_t<T>::type, typename primitive_expr_t<U>::type> and
6283 : : (primitive_expr_t<T>::num_simd_lanes == primitive_expr_t<U>::num_simd_lanes) and
6284 : : requires (PrimitiveExpr<common_type_t<typename primitive_expr_t<T>::type, typename primitive_expr_t<U>::type>,
6285 : : primitive_expr_t<T>::num_simd_lanes> e,
6286 : : const std::array<uint8_t, M> &a)
6287 : : { Module::Get().emit_shuffle_bytes(e, e, a); }
6288 : 8 : inline auto ShuffleBytes(T &&_first, U &&_second, const std::array<uint8_t, M> &indices)
6289 : : {
6290 : 8 : primitive_expr_t<T> first(std::forward<T>(_first));
6291 [ + - + - : 8 : primitive_expr_t<U> second(std::forward<U>(_second));
# # ]
6292 : :
6293 : : using To = common_type_t<typename decltype(first)::type, typename decltype(second)::type>;
6294 : 8 : constexpr std::size_t L = decltype(first)::num_simd_lanes;
6295 : :
6296 [ + - + - : 8 : return Module::Get().emit_shuffle_bytes<To, L>(first.template to<To, L>(), second.template to<To, L>(), indices);
+ - - + +
- + - + -
- + # # #
# # # #
# ]
6297 : 8 : }
6298 : :
6299 : : template<primitive_convertible T, primitive_convertible U, std::size_t M>
6300 : : requires have_common_type<typename primitive_expr_t<T>::type, typename primitive_expr_t<U>::type> and
6301 : : (primitive_expr_t<T>::num_simd_lanes == primitive_expr_t<U>::num_simd_lanes) and
6302 : : requires (PrimitiveExpr<common_type_t<typename primitive_expr_t<T>::type, typename primitive_expr_t<U>::type>,
6303 : : primitive_expr_t<T>::num_simd_lanes> e,
6304 : : const std::array<uint8_t, M> &a)
6305 : : { Module::Get().emit_shuffle_lanes(e, e, a); }
6306 : 0 : inline auto ShuffleLanes(T &&_first, U &&_second, const std::array<uint8_t, M> &indices)
6307 : : {
6308 : 0 : primitive_expr_t<T> first(std::forward<T>(_first));
6309 [ # # # # : 0 : primitive_expr_t<U> second(std::forward<U>(_second));
# # ]
6310 : :
6311 : : using To = common_type_t<typename decltype(first)::type, typename decltype(second)::type>;
6312 : 0 : constexpr std::size_t L = decltype(first)::num_simd_lanes;
6313 : :
6314 [ # # # # : 0 : return Module::Get().emit_shuffle_lanes<To, L>(first.template to<To, L>(), second.template to<To, L>(), indices);
# # # # #
# # # # #
# # # # #
# # # #
# ]
6315 : 0 : }
6316 : :
6317 : : template<expr_convertible T, expr_convertible U, std::size_t M>
6318 : : requires (not primitive_convertible<T> or not primitive_convertible<U>) and
6319 : : have_common_type<typename expr_t<T>::type, typename expr_t<U>::type> and
6320 : : (expr_t<T>::num_simd_lanes == expr_t<U>::num_simd_lanes) and
6321 : : requires (Expr<common_type_t<typename expr_t<T>::type, typename expr_t<U>::type>, expr_t<T>::num_simd_lanes> e,
6322 : : const std::array<uint8_t, M> &a)
6323 : : { Module::Get().emit_shuffle_bytes(e, e, a); }
6324 : : inline auto ShuffleBytes(T &&_first, U &&_second, const std::array<uint8_t, M> &indices)
6325 : : {
6326 : : expr_t<T> first(std::forward<T>(_first));
6327 : : expr_t<U> second(std::forward<U>(_second));
6328 : :
6329 : : using To = common_type_t<typename decltype(first)::type, typename decltype(second)::type>;
6330 : : constexpr std::size_t L = decltype(first)::num_simd_lanes;
6331 : :
6332 : : return Module::Get().emit_shuffle_bytes<To, L>(first.template to<To, L>(), second.template to<To, L>(), indices);
6333 : : }
6334 : :
6335 : : template<expr_convertible T, expr_convertible U, std::size_t M>
6336 : : requires (not primitive_convertible<T> or not primitive_convertible<U>) and
6337 : : have_common_type<typename expr_t<T>::type, typename expr_t<U>::type> and
6338 : : (expr_t<T>::num_simd_lanes == expr_t<U>::num_simd_lanes) and
6339 : : requires (Expr<common_type_t<typename expr_t<T>::type, typename expr_t<U>::type>, expr_t<T>::num_simd_lanes> e,
6340 : : const std::array<uint8_t, M> &a)
6341 : : { Module::Get().emit_shuffle_lanes(e, e, a); }
6342 : 4 : inline auto ShuffleLanes(T &&_first, U &&_second, const std::array<uint8_t, M> &indices)
6343 : : {
6344 : 4 : expr_t<T> first(std::forward<T>(_first));
6345 [ + - + - ]: 4 : expr_t<U> second(std::forward<U>(_second));
6346 : :
6347 : : using To = common_type_t<typename decltype(first)::type, typename decltype(second)::type>;
6348 : 4 : constexpr std::size_t L = decltype(first)::num_simd_lanes;
6349 : :
6350 [ + - + - : 4 : return Module::Get().emit_shuffle_lanes<To, L>(first.template to<To, L>(), second.template to<To, L>(), indices);
+ - - + +
- + - + -
- + ]
6351 : 4 : }
6352 : :
6353 : :
6354 : : /*----- If -----------------------------------------------------------------------------------------------------------*/
6355 : :
6356 : : struct If
6357 : : {
6358 : : using continuation_t = std::function<void(void)>;
6359 : :
6360 : : private:
6361 : : PrimitiveExpr<bool, 1> cond_;
6362 : : std::string name_;
6363 : :
6364 : : public:
6365 : : continuation_t Then, Else;
6366 : :
6367 : : template<primitive_convertible C>
6368 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6369 : 6080 : explicit If(C &&cond)
6370 : 3040 : : cond_(std::forward<C>(cond))
6371 [ + - - + : 3040 : , name_(Module::Unique_If_Name())
+ - - + #
# # # ]
6372 : 3040 : { }
6373 : :
6374 : : If(const If&) = delete;
6375 : : If(If&&) = delete;
6376 : :
6377 : : ~If();
6378 : : };
6379 : :
6380 : : /*----- Loop ---------------------------------------------------------------------------------------------------------*/
6381 : :
6382 : : /** Implements a loop which iterates exactly once unless explicitly `continue`-ed. The loop may be exited by
6383 : : * explicitly `break`-ing out of it. */
6384 : : struct Loop
6385 : : {
6386 : : friend void swap(Loop &first, Loop &second) {
6387 : : using std::swap;
6388 : : swap(first.body_, second.body_);
6389 : : swap(first.loop_, second.loop_);
6390 : : }
6391 : :
6392 : : private:
6393 : : Block body_; ///< the loop body
6394 : : ::wasm::Loop *loop_ = nullptr; ///< the Binaryen loop
6395 : :
6396 : : private:
6397 : : /** Convenience c'tor accessible via tag-dispatching. Expects an already unique \p name. */
6398 : 188 : Loop(std::string name, tag<int>)
6399 [ + - ]: 188 : : body_(name + ".body", false)
6400 [ + - + - : 188 : , loop_(M_notnull(Module::Builder().makeLoop(name, &body_.get())))
+ - + - +
- ]
6401 : : {
6402 [ + - + - ]: 376 : Module::Get().push_branch_targets(
6403 [ + - ]: 188 : /* brk= */ body_.get().name,
6404 : 188 : /* continu= */ loop_->name
6405 : : );
6406 : 188 : }
6407 : :
6408 : : public:
6409 [ + - - + ]: 188 : explicit Loop(std::string name) : Loop(Module::Unique_Loop_Name(name), tag<int>{}) { }
6410 [ + - - + ]: 88 : explicit Loop(const char *name) : Loop(std::string(name)) { }
6411 : :
6412 : : Loop(const Loop&) = delete;
6413 : : Loop(Loop &&other) { swap(*this, other); }
6414 : :
6415 : 188 : ~Loop() {
6416 [ - + ]: 188 : if (loop_) {
6417 [ + - + - ]: 188 : Module::Get().pop_branch_targets();
6418 [ + - + - ]: 188 : Module::Block().list.push_back(loop_);
6419 : 188 : }
6420 : 188 : }
6421 : :
6422 : : Loop & operator=(Loop &&other) { swap(*this, other); return *this; }
6423 : :
6424 : : std::string name() const { return loop_->name.toString(); }
6425 : :
6426 : 288 : Block & body() { return body_; }
6427 : : const Block & body() const { return body_; }
6428 : : };
6429 : :
6430 : : struct DoWhile : Loop
6431 : : {
6432 : : template<primitive_convertible C>
6433 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6434 : 100 : DoWhile(std::string name, C &&_cond)
6435 [ + - ]: 100 : : Loop(name)
6436 : : {
6437 [ + - ]: 100 : PrimitiveExpr<bool, 1> cond(std::forward<C>(_cond));
6438 : :
6439 : : /*----- Update condition in branch targets. -----*/
6440 [ + - + - ]: 100 : auto branch_targets = Module::Get().pop_branch_targets();
6441 [ + - + - : 100 : Module::Get().push_branch_targets(branch_targets.brk, branch_targets.continu, cond);
- + ]
6442 : 100 : }
6443 : :
6444 : : template<primitive_convertible C>
6445 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6446 [ + - - + ]: 40 : DoWhile(const char *name, C &&cond) : DoWhile(std::string(name), cond) { }
6447 : :
6448 : : DoWhile(const DoWhile&) = delete;
6449 : : DoWhile(DoWhile&&) = default;
6450 : :
6451 : : ~DoWhile();
6452 : : };
6453 : :
6454 : : struct While
6455 : : {
6456 : : private:
6457 : : PrimitiveExpr<bool, 1> cond_;
6458 : : std::unique_ptr<DoWhile> do_while_;
6459 : :
6460 : : public:
6461 : 60 : While(std::string name, PrimitiveExpr<bool, 1> cond)
6462 : 60 : : cond_(cond.clone())
6463 [ + - - + ]: 60 : , do_while_(std::make_unique<DoWhile>(name + ".do-while", cond))
6464 : 60 : { }
6465 : :
6466 : : template<primitive_convertible C>
6467 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6468 : : While(std::string name, C &&cond) : While(name, PrimitiveExpr<bool, 1>(std::forward<C>(cond))) { }
6469 : :
6470 : : template<primitive_convertible C>
6471 : : requires requires (C &&c) { PrimitiveExpr<bool, 1>(std::forward<C>(c)); }
6472 [ # # # # : 60 : While(const char *name, C &&cond) : While(std::string(name), cond) { }
# # + - +
- - + ]
6473 : :
6474 : : While(const While&) = delete;
6475 : : While(While&&) = default;
6476 : :
6477 : : ~While();
6478 : :
6479 : 60 : Block & body() { return do_while_->body(); }
6480 : : const Block & body() const { return do_while_->body(); }
6481 : : };
6482 : :
6483 : :
6484 : : /*======================================================================================================================
6485 : : * Allocator
6486 : : *====================================================================================================================*/
6487 : :
6488 : : struct Allocator
6489 : : {
6490 : : public:
6491 : 1654 : virtual ~Allocator() { }
6492 : :
6493 : : public:
6494 : : /** Pre-allocates memory for \p bytes consecutive bytes with alignment requirement \p align and returns a raw
6495 : : * pointer to the beginning of this memory. */
6496 : : virtual void * raw_allocate(uint32_t bytes, uint32_t align = 1) = 0;
6497 : : /** Pre-allocates memory for \p bytes consecutive bytes with alignment requirement \p align and returns a pointer
6498 : : * to the beginning of this memory. */
6499 : : virtual Ptr<void> pre_allocate(uint32_t bytes, uint32_t align = 1) = 0;
6500 : : /** Allocates memory for \p bytes consecutive bytes with alignment requirement \p align and returns a pointer to the
6501 : : * beginning of this memory. */
6502 : : virtual Var<Ptr<void>> allocate(U32x1 bytes, uint32_t align = 1) = 0;
6503 : : /** Deallocates the `bytes` consecutive bytes of allocated memory at address `ptr`. */
6504 : : virtual void deallocate(Ptr<void> ptr, U32x1 bytes) = 0;
6505 : :
6506 : : /** Performs the actual pre-allocations. Must be called exactly **once** **after** the last pre-allocation was
6507 : : * requested.
6508 : : * Returns the initial allocation address, i.e. an address after the last pre-allocated address. */
6509 : : virtual uint32_t perform_pre_allocations() = 0;
6510 : :
6511 : : /** Returns the pre-allocated memory overall consumption. */
6512 : : virtual uint32_t pre_allocated_memory_consumption() const = 0;
6513 : : /** Returns the allocated memory overall consumption. */
6514 : : virtual U32x1 allocated_memory_consumption() const = 0;
6515 : : /** Returns the allocated memory peak consumption. */
6516 : : virtual U32x1 allocated_memory_peak() const = 0;
6517 : :
6518 [ + - ]: 194 : Var<Ptr<void>> allocate(uint32_t bytes, uint32_t align = 1) { return allocate(U32x1(bytes), align); }
6519 [ + - - + ]: 2 : void deallocate(Ptr<void> ptr, uint32_t bytes) { return deallocate(ptr, U32x1(bytes)); }
6520 : :
6521 : :
6522 : : /** Pre-allocates memory for exactly one value of type \tparam T. Returns a raw pointer to this memory. */
6523 : : template<dsl_primitive T>
6524 : 0 : T * raw_malloc() { return raw_malloc<T>(1U); }
6525 : : /** Pre-allocates memory for exactly one value of type \tparam T and number of SIMD lanes \tparam L. Returns a
6526 : : * pointer to this memory. */
6527 : : template<dsl_primitive T, std::size_t L = 1>
6528 : 124 : Ptr<PrimitiveExpr<T, L>> pre_malloc() { return pre_malloc<T, L>(1U); }
6529 : : /** Allocates memory for exactly one value of type \tparam T and number of SIMD lanes \tparam L. Returns a
6530 : : * pointer to this memory. */
6531 : : template<dsl_primitive T, std::size_t L = 1>
6532 : 64 : Var<Ptr<PrimitiveExpr<T, L>>> malloc() { return malloc<T, L>(1U); }
6533 : :
6534 : : /** Pre-allocates memory for an array of \p count consecutive values of type \tparam T. Returns a raw pointer to
6535 : : * this memory. */
6536 : : template<dsl_primitive T>
6537 : 0 : T * raw_malloc(uint32_t count) { return static_cast<T*>(raw_allocate(sizeof(T) * count, alignof(T))); }
6538 : : /** Pre-allocates memory for an array of \p count consecutive values of type \tparam T and number of SIMD lanes
6539 : : * \tparam L. Returns a pointer to this memory. */
6540 : : template<dsl_primitive T, std::size_t L = 1>
6541 : 124 : Ptr<PrimitiveExpr<T, L>> pre_malloc(uint32_t count) {
6542 : : if constexpr (L == 1)
6543 [ # # ]: 0 : return pre_allocate(sizeof(T) * count, alignof(T)).template to<T*, L>();
6544 : : else if constexpr (L * sizeof(T) <= 16)
6545 [ + - + - : 124 : return pre_allocate(16 * count, alignof(T)).template to<T*, L>();
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
6546 : : else
6547 : : return pre_allocate(16 * PrimitiveExpr<T, L>::num_vectors * count, alignof(T)).template to<T*, L>();
6548 : 0 : }
6549 : : /** Allocates memory for an array of \p count consecutive values of type \tparam T and number of SIMD lanes
6550 : : * \tparam L. Returns a pointer to this memory. */
6551 : : template<dsl_primitive T, std::size_t L = 1, typename U>
6552 : : requires requires (U &&u) { U32x1(std::forward<U>(u)); }
6553 : 194 : Var<Ptr<PrimitiveExpr<T, L>>> malloc(U &&count) {
6554 : : if constexpr (L == 1) {
6555 [ + - + - : 194 : Var<Ptr<PrimitiveExpr<T, L>>> ptr(
+ - + - +
- + - +
- ]
6556 [ + - + - : 192 : allocate(uint32_t(sizeof(T)) * std::forward<U>(count), alignof(T)).template to<T*, L>()
+ - + - +
- + - + -
+ - ]
6557 : : );
6558 : 192 : return ptr;
6559 [ + - + - : 192 : } else if constexpr (L * sizeof(T) <= 16) {
+ - + - +
- + - +
- ]
6560 [ + - ]: 2 : Var<Ptr<PrimitiveExpr<T, L>>> ptr(
6561 [ + - ]: 2 : allocate(16U * std::forward<U>(count), alignof(T)).template to<T*, L>()
6562 : : );
6563 : 2 : return ptr;
6564 [ + - ]: 2 : } else {
6565 : : Var<Ptr<PrimitiveExpr<T, L>>> ptr(
6566 : : allocate(
6567 : : uint32_t(16 * PrimitiveExpr<T, L>::num_vectors) * std::forward<U>(count), alignof(T)
6568 : : ).template to<T*, L>()
6569 : : );
6570 : : return ptr;
6571 : : }
6572 : 194 : }
6573 : :
6574 : : /** Frees exactly one value of type \tparam T of allocated memory pointed by \p ptr. */
6575 : : template<primitive_convertible T>
6576 : : requires requires (T &&t) { primitive_expr_t<T>(std::forward<T>(t)).template to<void*>(); }
6577 : 2 : void free(T &&ptr) { free(std::forward<T>(ptr), 1U); }
6578 : :
6579 : : /** Frees \p count consecutive values of type \tparam T of allocated memory pointed by \p ptr. */
6580 : : template<primitive_convertible T, typename U>
6581 : : requires requires (U &&u) { U32x1(std::forward<U>(u)); } and
6582 : : requires (T &&t) { primitive_expr_t<T>(std::forward<T>(t)).template to<void*>(); }
6583 : 2 : void free(T &&ptr, U &&count) {
6584 : 2 : primitive_expr_t<T> _ptr(std::forward<T>(ptr));
6585 : : using pointed_type = typename decltype(_ptr)::pointed_type;
6586 : 2 : constexpr std::size_t L = decltype(_ptr)::num_simd_lanes;
6587 : : if constexpr (L == 1)
6588 [ + - - + : 2 : deallocate(_ptr.template to<void*>(), uint32_t(sizeof(pointed_type)) * std::forward<U>(count));
# # ]
6589 : : else if constexpr (L * sizeof(T) <= 16)
6590 : : deallocate(_ptr.template to<void*>(), 16U * std::forward<U>(count));
6591 : : else
6592 : : deallocate(_ptr.template to<void*>(),
6593 : : uint32_t(16 * PrimitiveExpr<T, L>::num_vectors) * std::forward<U>(count));
6594 : 2 : }
6595 : : };
6596 : :
6597 : :
6598 : : /*######################################################################################################################
6599 : : * DELAYED DEFINITIONS
6600 : : *####################################################################################################################*/
6601 : :
6602 : : /*======================================================================================================================
6603 : : * Module
6604 : : *====================================================================================================================*/
6605 : :
6606 : 2836 : inline void Module::create_local_bitmap_stack()
6607 : : {
6608 : 2836 : local_bitmaps_stack_.emplace_back();
6609 : 2836 : }
6610 : :
6611 : 2836 : inline void Module::create_local_bitvector_stack()
6612 : : {
6613 : 2836 : local_bitvectors_stack_.emplace_back();
6614 : 2836 : }
6615 : :
6616 : 2836 : inline void Module::dispose_local_bitmap_stack()
6617 : : {
6618 : 2836 : auto &local_bitmaps = local_bitmaps_stack_.back();
6619 [ + + ]: 3020 : for (LocalBitmap *bitmap : local_bitmaps) {
6620 : 184 : M_insist(~bitmap->bitmask == 0, "all bits must have been deallocated");
6621 [ - + ]: 184 : delete bitmap;
6622 : : }
6623 : 2836 : local_bitmaps_stack_.pop_back();
6624 : 2836 : }
6625 : :
6626 : 2836 : inline void Module::dispose_local_bitvector_stack()
6627 : : {
6628 : 2836 : auto &local_bitvectors = local_bitvectors_stack_.back();
6629 [ + + ]: 2852 : for (LocalBitvector *bitvector : local_bitvectors) {
6630 : : #ifndef NDEBUG
6631 [ + + ]: 144 : for (auto bitmask : bitvector->bitmask_per_offset)
6632 : 128 : M_insist(uint16_t(~bitmask) == 0, "all bits must have been deallocated");
6633 : : #endif
6634 [ - + ]: 16 : delete bitvector;
6635 : : }
6636 : 2836 : local_bitvectors_stack_.pop_back();
6637 : 2836 : }
6638 : :
6639 : : template<std::size_t L>
6640 : : requires (L > 0) and (L <= 16)
6641 : 628 : inline LocalBit<L> Module::allocate_bit()
6642 : : {
6643 : : if constexpr (L == 1) {
6644 : 596 : auto &local_bitmaps = local_bitmaps_stack_.back();
6645 : :
6646 [ + + ]: 596 : if (local_bitmaps.empty())
6647 [ + - ]: 184 : local_bitmaps.emplace_back(new LocalBitmap()); // allocate new local bitmap in current function
6648 : :
6649 : 596 : LocalBitmap &bitmap = *local_bitmaps.back();
6650 : 596 : M_insist(bitmap.bitmask, "bitmap must have at least one bit unoccupied");
6651 : :
6652 : 596 : uint8_t bit_offset = std::countr_zero(bitmap.bitmask);
6653 : 596 : bitmap.bitmask ^= 1UL << bit_offset; // clear allocated bit
6654 : :
6655 : 596 : LocalBit<L> bit(bitmap, bit_offset);
6656 : :
6657 [ + - ]: 596 : if (bitmap.bitmask == 0) // all bits have been allocated
6658 : 0 : local_bitmaps.pop_back(); // remove bitmap entry, ownership transitions to *all* referencing `LocalBit`s
6659 : :
6660 : 596 : return bit;
6661 [ + - ]: 596 : } else {
6662 : 32 : bool fresh_bitvector = false;
6663 : :
6664 : 32 : auto &local_bitvectors = local_bitvectors_stack_.back();
6665 : :
6666 [ + + # # : 32 : if (local_bitvectors.empty()) {
+ - + + ]
6667 : : allocate_bitvector:
6668 [ - + # # : 16 : local_bitvectors.emplace_back(new LocalBitvector()); // allocate new local bitvector in current function
# # - + ]
6669 : 16 : fresh_bitvector = true;
6670 : 16 : }
6671 : :
6672 : 32 : LocalBitvector &bitvector = *local_bitvectors.back();
6673 : :
6674 : : uint8_t bit_offset;
6675 : 32 : uint8_t starting_lane = uint8_t(-1U);
6676 : 32 : constexpr uint16_t MASK = (1U << L) - 1U;
6677 [ + - # # : 32 : for (uint8_t lane = 0; lane <= 16 - L; ++lane) {
+ - + - ]
6678 : 32 : bit_offset = 0;
6679 [ + - # # : 52 : for (uint16_t bitmask : bitvector.bitmask_per_offset) {
+ - + - ]
6680 : 52 : const uint16_t masked = bitmask bitand (MASK << lane);
6681 [ + + # # : 52 : if (masked == (MASK << lane)) { // `bit_offset`-th bit is free in L consecutive lanes
+ + + + ]
6682 : 32 : starting_lane = lane;
6683 : 32 : goto found_bits;
6684 : : }
6685 : 20 : ++bit_offset;
6686 : : }
6687 : 0 : }
6688 : : found_bits:
6689 [ - + # # : 32 : if (starting_lane == uint8_t(-1U)) {
- + - + ]
6690 : 0 : M_insist(not fresh_bitvector, "fresh bitvector must have at least L consecutive bits unoccupied");
6691 : 0 : goto allocate_bitvector; // no bits found, retry with fresh bitvector
6692 : : }
6693 : :
6694 : 32 : bitvector.bitmask_per_offset[bit_offset] ^= MASK << starting_lane; // clear allocated bits
6695 : :
6696 : 32 : LocalBit<L> bit(bitvector, bit_offset, starting_lane);
6697 : :
6698 : 32 : const auto &bitmasks = bitvector.bitmask_per_offset;
6699 : 92 : auto pred = [](auto bitmask){ return bitmask == 0; };
6700 [ + - + - : 32 : if (std::all_of(bitmasks.cbegin(), bitmasks.cend(), pred)) // all bits have been allocated
# # # # +
- + - + -
+ - ]
6701 : 0 : local_bitvectors.pop_back(); // remove bitvector entry, ownership transitions to *all* referencing `LocalBit`s
6702 : :
6703 : 32 : return bit;
6704 [ + - # # : 32 : }
+ - + - ]
6705 : 628 : }
6706 : :
6707 : : template<typename T, std::size_t L>
6708 : : requires (L * sizeof(T) <= 16)
6709 : 1674 : inline PrimitiveExpr<T, L> Module::get_global(const char *name)
6710 : : {
6711 [ + - + - ]: 1674 : return PrimitiveExpr<T, L>(builder_.makeGlobalGet(name, wasm_type<T, L>()));
6712 : 0 : }
6713 : :
6714 : : template<typename ReturnType, typename... ParamTypes, std::size_t... ParamLs>
6715 : : requires std::is_void_v<ReturnType>
6716 : 263 : inline void Module::emit_call(const char *fn, PrimitiveExpr<ParamTypes, ParamLs>... args)
6717 : : {
6718 [ + - # # ]: 526 : active_block_->list.push_back(
6719 [ + - + - : 263 : builder_.makeCall(fn, { args.expr()... }, wasm_type<ReturnType, 1>())
+ - # # #
# # # ]
6720 : : );
6721 : 263 : }
6722 : :
6723 : : template<typename ReturnType, std::size_t ReturnL, typename... ParamTypes, std::size_t... ParamLs>
6724 : : requires dsl_primitive<ReturnType> or dsl_pointer_to_primitive<ReturnType>
6725 : 0 : inline PrimitiveExpr<ReturnType, ReturnL> Module::emit_call(const char *fn, PrimitiveExpr<ParamTypes, ParamLs>... args)
6726 : : {
6727 [ # # # # : 0 : return PrimitiveExpr<ReturnType, ReturnL>(
# # # # #
# # # # #
# # ]
6728 [ # # # # : 0 : builder_.makeCall(fn, { args.expr()... }, wasm_type<ReturnType, ReturnL>())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6729 : : );
6730 : 0 : }
6731 : :
6732 : : inline void Module::emit_return()
6733 : : {
6734 : : active_block_->list.push_back(builder_.makeReturn());
6735 : : }
6736 : :
6737 : : template<typename T, std::size_t L>
6738 : 2457 : inline void Module::emit_return(PrimitiveExpr<T, L> value)
6739 : : {
6740 : 2457 : active_block_->list.push_back(builder_.makeReturn(value.expr()));
6741 : 2457 : }
6742 : :
6743 : : template<typename T, std::size_t L>
6744 : 684 : inline void Module::emit_return(Expr<T, L> value)
6745 : : {
6746 [ + - + - : 684 : emit_return(value.insist_not_null());
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
6747 : 684 : }
6748 : :
6749 : : /** Emit an unconditional break, breaking \p level levels. */
6750 : 130 : inline void Module::emit_break(std::size_t level)
6751 : : {
6752 : 130 : M_insist(level > 0);
6753 : 130 : M_insist(branch_target_stack_.size() >= level);
6754 : 130 : auto &branch_targets = branch_target_stack_[branch_target_stack_.size() - level];
6755 : 130 : active_block_->list.push_back(builder_.makeBreak(branch_targets.brk));
6756 : 130 : }
6757 : :
6758 : : /** Emit a conditional break, breaking if \p cond is `true` and breaking \p level levels. */
6759 : 180 : inline void Module::emit_break(PrimitiveExpr<bool, 1> cond, std::size_t level)
6760 : : {
6761 : 180 : M_insist(level > 0);
6762 : 180 : M_insist(branch_target_stack_.size() >= level);
6763 : 180 : auto &branch_targets = branch_target_stack_[branch_target_stack_.size() - level];
6764 : 180 : active_block_->list.push_back(builder_.makeBreak(branch_targets.brk, nullptr, cond.expr()));
6765 : 180 : }
6766 : :
6767 : : template<typename T, std::size_t L>
6768 : 384 : PrimitiveExpr<T, L> Module::emit_select(PrimitiveExpr<bool, 1> cond, PrimitiveExpr<T, L> tru, PrimitiveExpr<T, L> fals)
6769 : : {
6770 : : if constexpr (L * sizeof(T) <= 16) {
6771 : 384 : auto referenced_bits = cond.referenced_bits(); // moved
6772 [ # # + - : 384 : referenced_bits.splice(referenced_bits.end(), tru.referenced_bits());
# # # # #
# # # # #
# # ]
6773 [ # # + - : 384 : referenced_bits.splice(referenced_bits.end(), fals.referenced_bits());
# # # # #
# # # # #
# # ]
6774 [ - + - + : 384 : return PrimitiveExpr<T, L>(
# # - + #
# # # # #
- + # # #
# # # # #
# # # # #
# # # #
# ]
6775 [ + - + - : 384 : /* expr= */ builder_.makeSelect(cond.expr(), tru.expr(), fals.expr()),
+ - + - +
- + - + -
+ - # # #
# # # # #
+ - + - +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6776 : 384 : /* referenced_bits= */ std::move(referenced_bits)
6777 : : );
6778 : 384 : } else {
6779 : : using ResT = PrimitiveExpr<T, L>;
6780 : : std::array<typename ResT::vector_type, ResT::num_vectors> vectors;
6781 : : auto vectors_tru = tru.vectors();
6782 : : auto vectors_fals = fals.vectors();
6783 : : for (std::size_t idx = 0; idx < ResT::num_vectors; ++idx) {
6784 : : auto cond_cpy = cond.clone();
6785 : : auto referenced_bits = cond_cpy.referenced_bits(); // moved
6786 : : referenced_bits.splice(referenced_bits.end(), vectors_tru[idx].referenced_bits());
6787 : : referenced_bits.splice(referenced_bits.end(), vectors_fals[idx].referenced_bits());
6788 : : vectors[idx] = typename ResT::vector_type(
6789 : : /* expr= */ builder_.makeSelect(cond_cpy.expr(), vectors_tru[idx].expr(),
6790 : : vectors_fals[idx].expr()),
6791 : : /* referenced_bits= */ std::move(referenced_bits)
6792 : : );
6793 : : }
6794 : : cond.discard(); // since it was always cloned
6795 : : return ResT(std::move(vectors));
6796 : : }
6797 : 384 : }
6798 : :
6799 : : template<typename T, std::size_t L>
6800 : 0 : Expr<T, L> Module::emit_select(PrimitiveExpr<bool, 1> cond, Expr<T, L> tru, Expr<T, L> fals)
6801 : : {
6802 [ # # # # : 0 : if (tru.can_be_null() or fals.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6803 : 0 : auto [tru_val, tru_is_null] = tru.split();
6804 [ # # # # : 0 : auto [fals_val, fals_is_null] = fals.split();
# # # # #
# # # #
# ]
6805 [ # # # # : 0 : auto cond_cpy = cond.clone();
# # # # #
# # # #
# ]
6806 [ # # # # : 0 : return Expr<T, L>(
# # # # #
# # # #
# ]
6807 [ # # # # : 0 : /* value= */ emit_select(cond, tru_val, fals_val),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6808 [ # # # # : 0 : /* is_null= */ emit_select(cond_cpy, tru_is_null, fals_is_null)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6809 : : );
6810 : 0 : } else {
6811 [ # # # # : 0 : return Expr<T, L>(
# # # # #
# # # #
# ]
6812 [ # # # # : 0 : /* value= */ emit_select(cond, tru.insist_not_null(), fals.insist_not_null())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6813 : : );
6814 : : }
6815 : 0 : }
6816 : :
6817 : : template<typename T, std::size_t L>
6818 : : requires (L > 1) and requires (PrimitiveExpr<int8_t, L> e) { e.template to<int_t<sizeof(T)>, L>(); }
6819 : 6 : PrimitiveExpr<T, L> Module::emit_select(PrimitiveExpr<bool, L> cond, PrimitiveExpr<T, L> tru, PrimitiveExpr<T, L> fals)
6820 : : {
6821 : : using To = int_t<sizeof(T)>;
6822 : :
6823 [ + - # # : 6 : PrimitiveExpr<int8_t, L> mask_i8(cond.template move<int8_t, L>()); // convert without transforming `true`, i.e. 0xff, to 1
# # # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6824 [ + - # # : 6 : PrimitiveExpr<To, L> mask = mask_i8.template to<To, L>(); // convert (w/ sign extension!) to same bit width as values
# # # # #
# + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6825 : :
6826 : : if constexpr (L * sizeof(T) <= 16) {
6827 : 2 : auto referenced_bits = mask.referenced_bits(); // moved
6828 : 2 : referenced_bits.splice(referenced_bits.end(), tru.referenced_bits());
6829 : 2 : referenced_bits.splice(referenced_bits.end(), fals.referenced_bits());
6830 [ - + # # : 2 : return PrimitiveExpr<T, L>(
# # # # #
# # # # #
# # # # #
# ]
6831 [ + - + - : 4 : /* expr= */ builder_.makeSIMDTernary(::wasm::SIMDTernaryOp::Bitselect, tru.expr(), fals.expr(),
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
6832 [ + - # # : 2 : mask.expr()),
# # # # #
# # # # #
# # # # #
# ]
6833 : 2 : /* referenced_bits= */ std::move(referenced_bits)
6834 : : );
6835 : 2 : } else {
6836 : : using ResT = PrimitiveExpr<T, L>;
6837 : 4 : std::array<typename ResT::vector_type, ResT::num_vectors> vectors;
6838 [ + - # # : 4 : auto vectors_mask = mask.vectors();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6839 : : static_assert(ResT::num_vectors == vectors_mask.size());
6840 [ + - # # : 4 : auto vectors_tru = tru.vectors();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6841 [ + - # # : 4 : auto vectors_fals = fals.vectors();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6842 [ + + # # : 12 : for (std::size_t idx = 0; idx < ResT::num_vectors; ++idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6843 : 8 : auto referenced_bits = vectors_mask[idx].referenced_bits(); // moved
6844 : 8 : referenced_bits.splice(referenced_bits.end(), vectors_tru[idx].referenced_bits());
6845 : 8 : referenced_bits.splice(referenced_bits.end(), vectors_fals[idx].referenced_bits());
6846 [ - + # # : 8 : vectors[idx] = typename ResT::vector_type(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6847 [ + - # # : 16 : /* expr= */ builder_.makeSIMDTernary(::wasm::SIMDTernaryOp::Bitselect,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6848 [ + - + - : 8 : vectors_tru[idx].expr(), vectors_fals[idx].expr(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6849 [ + - # # : 8 : vectors_mask[idx].expr()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6850 : 8 : /* referenced_bits= */ std::move(referenced_bits)
6851 : : );
6852 : 8 : }
6853 [ + - - + : 4 : return ResT(std::move(vectors));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6854 : 4 : }
6855 : 6 : }
6856 : :
6857 : : template<typename T, std::size_t L>
6858 : : requires (L > 1) and requires (PrimitiveExpr<int8_t, L> e) { e.template to<int_t<sizeof(T)>, L>(); }
6859 : 0 : Expr<T, L> Module::emit_select(PrimitiveExpr<bool, L> cond, Expr<T, L> tru, Expr<T, L> fals)
6860 : : {
6861 [ # # # # : 0 : if (tru.can_be_null() or fals.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6862 : 0 : auto [tru_val, tru_is_null] = tru.split();
6863 [ # # # # : 0 : auto [fals_val, fals_is_null] = fals.split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6864 [ # # # # : 0 : auto cond_cpy = cond.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6865 [ # # # # : 0 : return Expr<T, L>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6866 [ # # # # : 0 : /* value= */ emit_select(cond, tru_val, fals_val),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
6867 [ # # # # : 0 : /* is_null= */ emit_select(cond_cpy, tru_is_null, fals_is_null)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
6868 : : );
6869 : 0 : } else {
6870 [ # # # # : 0 : return Expr<T, L>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6871 [ # # # # : 0 : /* value= */ emit_select(cond, tru.insist_not_null(), fals.insist_not_null())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6872 : : );
6873 : : }
6874 : 0 : }
6875 : :
6876 : : template<typename T, std::size_t L, std::size_t M>
6877 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (M <= 16) and (M % sizeof(T) == 0)
6878 : : inline PrimitiveExpr<T, M / sizeof(T)>
6879 : 8 : Module::emit_shuffle_bytes(PrimitiveExpr<T, L> first, PrimitiveExpr<T, L> second,
6880 : : const std::array<uint8_t, M> &_indices)
6881 : : {
6882 : : std::array<uint8_t, 16> indices;
6883 [ + + + + : 92 : for (std::size_t idx = 0; idx < M; ++idx) {
# # ]
6884 [ + + + + : 84 : if (_indices[idx] < L * sizeof(T))
# # ]
6885 : 46 : indices[idx] = _indices[idx]; // given byte of `first`
6886 [ + - + - : 38 : else if (_indices[idx] < 2 * L * sizeof(T))
# # ]
6887 : 38 : indices[idx] = _indices[idx] + (16 - L * sizeof(T)); // shift to given byte of `second`
6888 : : else
6889 : 0 : indices[idx] = 15; // last byte of `first`
6890 : 84 : }
6891 : 8 : std::fill(indices.begin() + M, indices.end(), 15); // last byte of `first`
6892 : :
6893 : 8 : auto referenced_bits = first.referenced_bits(); // moved
6894 [ # # ]: 8 : referenced_bits.splice(referenced_bits.end(), second.referenced_bits());
6895 [ + - + - : 8 : auto vec = PrimitiveExpr<T, M / sizeof(T)>(
# # ]
6896 [ + - + - : 8 : /* expr= */ builder_.makeSIMDShuffle(first.expr(), second.expr(), indices),
+ - + - +
- + - # #
# # # # ]
6897 : 8 : /* referenced_bits= */ std::move(referenced_bits)
6898 : : );
6899 [ - + - + : 16 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
# # ]
6900 : : vec.template extract_unsafe<0>(), // extract a single value from vector to scalar
6901 : : vec);
6902 : 8 : }
6903 : :
6904 : : template<typename T, std::size_t L, std::size_t M>
6905 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (is_pow_2(M)) and (M * sizeof(T) <= 16)
6906 : : inline PrimitiveExpr<T, M>
6907 : 8 : Module::emit_shuffle_lanes(PrimitiveExpr<T, L> first, PrimitiveExpr<T, L> second,
6908 : : const std::array<uint8_t, M> &_indices)
6909 : : {
6910 : : std::array<uint8_t, 16> indices;
6911 [ + + # # : 28 : for (std::size_t idx = 0; idx < M; ++idx) {
+ + # # +
+ ]
6912 [ + + # # : 108 : for (std::size_t byte = 0; byte < sizeof(T); ++byte) {
+ + # # +
+ ]
6913 [ + + # # : 88 : if (_indices[idx] < L)
+ + # # +
+ ]
6914 : 44 : indices[idx * sizeof(T) + byte] = _indices[idx] * sizeof(T) + byte; // given lane of `first`
6915 [ + - # # : 44 : else if (_indices[idx] < 2 * L)
+ - # # +
- ]
6916 : 44 : indices[idx * sizeof(T) + byte] =
6917 : 44 : _indices[idx] * sizeof(T) + byte + (16 - L * sizeof(T)); // shift to given lane of `second`
6918 : : else
6919 : 0 : indices[idx * sizeof(T) + byte] = 15; // last byte of `first`
6920 : 88 : }
6921 : 20 : }
6922 : 8 : std::fill(indices.begin() + M * sizeof(T), indices.end(), 15); // last byte of `first`
6923 : :
6924 : 8 : auto referenced_bits = first.referenced_bits(); // moved
6925 : 8 : referenced_bits.splice(referenced_bits.end(), second.referenced_bits());
6926 [ + - # # : 8 : auto vec = PrimitiveExpr<T, M>(
+ - # # +
- ]
6927 [ + - + - : 8 : /* expr= */ builder_.makeSIMDShuffle(first.expr(), second.expr(), indices),
+ - # # #
# # # + -
+ - + - #
# # # # #
+ - + - +
- ]
6928 : 8 : /* referenced_bits= */ std::move(referenced_bits)
6929 : : );
6930 [ - + # # : 16 : return M_CONSTEXPR_COND(decltype(vec)::num_simd_lanes == 1,
- + # # -
+ ]
6931 : : vec.template extract_unsafe<0>(), // extract a single value from vector to scalar
6932 : : vec);
6933 : 8 : }
6934 : :
6935 : : template<typename T, std::size_t L, std::size_t M>
6936 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (M <= 16) and (M % sizeof(T) == 0) and (sizeof(T) == 1)
6937 : : inline Expr<T, M / sizeof(T)>
6938 : : Module::emit_shuffle_bytes(Expr<T, L> first, Expr<T, L> second, const std::array<uint8_t, M> &indices)
6939 : : {
6940 : : if (first.can_be_null() or second.can_be_null()) {
6941 : : auto [first_val, first_is_null] = first.split();
6942 : : auto [second_val, second_is_null] = second.split();
6943 : : return Expr<T, M / sizeof(T)>(
6944 : : /* value= */ emit_shuffle_bytes(first_val, second_val, indices),
6945 : : /* is_null= */ emit_shuffle_bytes(first_is_null, second_is_null, indices)
6946 : : );
6947 : : } else {
6948 : : return Expr<T, M / sizeof(T)>(
6949 : : /* value= */ emit_shuffle_bytes(first.insist_not_null(), second.insist_not_null(), indices)
6950 : : );
6951 : : }
6952 : : }
6953 : :
6954 : : template<typename T, std::size_t L, std::size_t M>
6955 : : requires (L > 1) and (L * sizeof(T) <= 16) and (M > 0) and (is_pow_2(M)) and (M * sizeof(T) <= 16)
6956 : : inline Expr<T, M>
6957 : 4 : Module::emit_shuffle_lanes(Expr<T, L> first, Expr<T, L> second, const std::array<uint8_t, M> &indices)
6958 : : {
6959 [ + - - + : 4 : if (first.can_be_null() or second.can_be_null()) {
+ - - + ]
6960 : 0 : auto [first_val, first_is_null] = first.split();
6961 [ # # # # ]: 0 : auto [second_val, second_is_null] = second.split();
6962 [ # # # # ]: 0 : return Expr<T, M>(
6963 [ # # # # : 0 : /* value= */ emit_shuffle_lanes(first_val, second_val, indices),
# # # # #
# # # ]
6964 [ # # # # : 0 : /* is_null= */ emit_shuffle_lanes(first_is_null, second_is_null, indices)
# # # # #
# # # ]
6965 : : );
6966 : 0 : } else {
6967 [ - + - + ]: 4 : return Expr<T, M>(
6968 [ + - + - : 4 : /* value= */ emit_shuffle_lanes(first.insist_not_null(), second.insist_not_null(), indices)
+ - + - ]
6969 : : );
6970 : : }
6971 : 4 : }
6972 : :
6973 : 100 : inline void Module::push_branch_targets(::wasm::Name brk, ::wasm::Name continu, PrimitiveExpr<bool, 1> condition)
6974 : : {
6975 : 100 : branch_target_stack_.emplace_back(brk, continu, condition.expr());
6976 : 100 : }
6977 : :
6978 : :
6979 : : /*======================================================================================================================
6980 : : * Block
6981 : : *====================================================================================================================*/
6982 : :
6983 : 0 : inline void Block::go_to(PrimitiveExpr<bool, 1> cond) const
6984 : : {
6985 : 0 : Module::Block().list.push_back(Module::Builder().makeBreak(get().name, nullptr, cond.expr()));
6986 : 0 : }
6987 : :
6988 : :
6989 : : /*======================================================================================================================
6990 : : * Specialization of `variable_storage` for local, non-`NULL` boolean.
6991 : : *====================================================================================================================*/
6992 : :
6993 : : namespace detail {
6994 : :
6995 : : /*----- Specialization for local variables of boolean type that *cannot* be `NULL`. ----------------------------------*/
6996 : :
6997 : : template<std::size_t L>
6998 : 624 : inline variable_storage<bool, VariableKind::Local, false, L>::variable_storage()
6999 : 1248 : : values_([](){
7000 : 624 : std::array<std::shared_ptr<LocalBit<bit_num_simd_lanes>>, num_locals> values;
7001 [ + + + + : 1252 : for (std::size_t idx = 0; idx < num_locals; ++idx)
+ + # # +
+ + + ]
7002 [ - + - + : 628 : values[idx] = std::make_shared<LocalBit<bit_num_simd_lanes>>(
- + # # -
+ - + ]
7003 [ + - + - : 628 : Module::Get().allocate_bit<bit_num_simd_lanes>()
+ - + - +
- + - # #
# # + - +
- + - +
- ]
7004 : : );
7005 : 624 : return values;
7006 : 624 : }())
7007 : 624 : { }
7008 : :
7009 : : template<std::size_t L>
7010 : 0 : void variable_storage<bool, VariableKind::Local, false, L>::set_true()
7011 : : {
7012 [ # # ]: 0 : for (auto &local_bit : values_)
7013 : 0 : local_bit->set();
7014 : 0 : }
7015 : :
7016 : : template<std::size_t L>
7017 : 0 : void variable_storage<bool, VariableKind::Local, false, L>::set_false()
7018 : : {
7019 [ # # # # : 0 : for (auto &local_bit : values_)
# # # # #
# # # ]
7020 : 0 : local_bit->clear();
7021 : 0 : }
7022 : :
7023 : : template<std::size_t L>
7024 : : template<primitive_convertible U>
7025 : : requires requires (U &&u) { PrimitiveExpr<bool, L>(primitive_expr_t<U>(std::forward<U>(u))); }
7026 : 652 : void variable_storage<bool, VariableKind::Local, false, L>::operator=(U &&_value)
7027 : : {
7028 : : if constexpr (num_locals == 1) {
7029 : 648 : PrimitiveExpr<bool, L> value(primitive_expr_t<U>(std::forward<U>(_value)));
7030 [ + - - + : 648 : values_[0]->set(value);
+ - - + #
# # # + -
- + # # #
# + - - +
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
7031 : 648 : } else {
7032 : 4 : PrimitiveExpr<bool, L> value(primitive_expr_t<U>(std::forward<U>(_value)));
7033 : : static_assert(num_locals == decltype(value)::num_vectors);
7034 [ # # + - : 4 : auto vectors = value.vectors();
# # ]
7035 [ # # + + : 12 : for (std::size_t idx = 0; idx < num_locals; ++idx)
# # ]
7036 [ # # # # : 8 : values_[idx]->set(vectors[idx]);
+ - + - #
# # # ]
7037 : 4 : }
7038 : 652 : }
7039 : :
7040 : : template<std::size_t L>
7041 : 640 : inline variable_storage<bool, VariableKind::Local, false, L>::operator PrimitiveExpr<bool, L>() const
7042 : : {
7043 : : if constexpr (num_locals == 1) {
7044 [ + - + - : 636 : return PrimitiveExpr<bool, L>(/* expr= */ values_[0]->is_set().expr(), /* referenced_bits= */ { values_[0] });
- + + - +
- - + # #
# # # # +
- + - - +
+ - + - -
+ ]
7045 : : } else {
7046 : : static_assert(num_locals == PrimitiveExpr<bool, L>::num_vectors);
7047 : 4 : std::array<typename PrimitiveExpr<bool, L>::vector_type, num_locals> vectors;
7048 [ + + ]: 12 : for (std::size_t idx = 0; idx < num_locals; ++idx)
7049 [ - + ]: 8 : vectors[idx] = typename PrimitiveExpr<bool, L>::vector_type(
7050 [ + - + - ]: 8 : /* expr= */ values_[idx]->is_set().expr(),
7051 [ - + ]: 8 : /* referenced_bits= */ { values_[idx] }
7052 : : );
7053 [ + - + - ]: 4 : return PrimitiveExpr<bool, L>(std::move(vectors));
7054 : 4 : }
7055 : 4 : }
7056 : :
7057 : : }
7058 : :
7059 : : #undef UNARY_LIST
7060 : : #undef BINARY_LIST
7061 : : #undef ASSIGNOP_LIST
7062 : :
7063 : :
7064 : : /*======================================================================================================================
7065 : : * explicit instantiation declarations
7066 : : *====================================================================================================================*/
7067 : :
7068 : : extern template void Module::emit_insist(PrimitiveExpr<bool, 2>, const char*, unsigned, const char*);
7069 : : extern template void Module::emit_insist(PrimitiveExpr<bool, 4>, const char*, unsigned, const char*);
7070 : : extern template void Module::emit_insist(PrimitiveExpr<bool, 8>, const char*, unsigned, const char*);
7071 : : extern template void Module::emit_insist(PrimitiveExpr<bool, 16>, const char*, unsigned, const char*);
7072 : : extern template void Module::emit_insist(PrimitiveExpr<bool, 32>, const char*, unsigned, const char*);
7073 : :
7074 : : }
7075 : :
7076 : : }
|
