Branch data Line data Source code
1 : : #include "backend/WasmUtil.hpp"
2 : :
3 : : #include "backend/Interpreter.hpp"
4 : : #include "backend/WasmMacro.hpp"
5 : : #include "mutable/util/macro.hpp"
6 : : #include <mutable/util/concepts.hpp>
7 : : #include <optional>
8 : : #include <regex>
9 : : #include <tuple>
10 : :
11 : :
12 : : using namespace m;
13 : : using namespace m::storage;
14 : : using namespace m::wasm;
15 : :
16 : :
17 : : namespace {
18 : :
19 : : namespace options {
20 : :
21 : : /** Whether data layout compilation makes use of pointer sharing optimization. */
22 : : bool pointer_sharing = true;
23 : :
24 : : /** Whether data layout compilation makes use of remainder removal optimization. */
25 : : bool remainder_removal = true;
26 : :
27 : : }
28 : :
29 : : __attribute__((constructor(201)))
30 : 1 : static void add_wasm_util_args()
31 : : {
32 : 1 : Catalog &C = Catalog::Get();
33 : :
34 : : /*----- Command-line arguments -----*/
35 : 1 : C.arg_parser().add<bool>(
36 : : /* group= */ "Wasm",
37 : : /* short= */ nullptr,
38 : : /* long= */ "--no-pointer-sharing",
39 : : /* description= */ "do not use pointer sharing optimization for data layout compilation",
40 : 0 : /* callback= */ [](bool){ options::pointer_sharing = false; }
41 : : );
42 : 1 : C.arg_parser().add<bool>(
43 : : /* group= */ "Wasm",
44 : : /* short= */ nullptr,
45 : : /* long= */ "--no-remainder-removal",
46 : : /* description= */ "do not use remainder removal optimization for data layout compilation",
47 : 0 : /* callback= */ [](bool){ options::remainder_removal = false; }
48 : : );
49 : 1 : }
50 : :
51 : : }
52 : :
53 : :
54 : : /*======================================================================================================================
55 : : * Helper functions
56 : : *====================================================================================================================*/
57 : :
58 : : /** Convert \p operand of some `SQL_t` type to the target type \tparam T. \tparam T must be a `SQL_t` type. Conversion
59 : : * is done *in place*, i.e. the `SQL_t` instance is directly modified. */
60 : : template<arithmetic T>
61 : 0 : void convert_in_place(SQL_t &operand)
62 : : {
63 : 0 : std::visit(overloaded {
64 : 0 : [&operand](auto &&actual) -> void requires requires { { actual.template to<T>() } -> sql_type; } {
65 : 0 : auto v = actual.template to<T>();
66 : 0 : operand.~SQL_t();
67 [ # # # # : 0 : new (&operand) SQL_t(v);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
68 : 0 : },
69 : 0 : [](auto &actual) -> void requires (not requires { { actual.template to<T>() } -> sql_type; }) {
70 : 0 : M_unreachable("illegal conversion");
71 : : },
72 : 0 : [](std::monostate) -> void { M_unreachable("invalid variant"); },
73 : 0 : }, operand);
74 : 1 : }
75 : :
76 : : /** Convert \p operand to runtime type \p to_type. This is done by delegating to `convert_in_place<T>` through a
77 : : * dynamic dispatch based on \p to_type. */
78 : 0 : void convert_in_place(SQL_t &operand, const Numeric *to_type)
79 : : {
80 [ # # # # ]: 0 : switch (to_type->kind) {
81 : : case Numeric::N_Decimal:
82 : 0 : M_unreachable("currently not supported");
83 : :
84 : : case Numeric::N_Int:
85 [ # # # # : 0 : switch (to_type->size()) {
# ]
86 : : default:
87 : 0 : M_unreachable("invalid integer size");
88 : : case 8:
89 : 0 : convert_in_place<int8_t>(operand);
90 : 0 : return;
91 : : case 16:
92 : 0 : convert_in_place<int16_t>(operand);
93 : 0 : return;
94 : : case 32:
95 : 0 : convert_in_place<int32_t>(operand);
96 : 0 : return;
97 : : case 64:
98 : 0 : convert_in_place<int64_t>(operand);
99 : 0 : return;
100 : : }
101 : : break;
102 : : case Numeric::N_Float:
103 [ # # ]: 0 : if (to_type->size() <= 32)
104 : 0 : convert_in_place<float>(operand);
105 : : else
106 : 0 : convert_in_place<double>(operand);
107 : 0 : break;
108 : : }
109 : 0 : }
110 : :
111 : : template<bool CanBeNull, std::size_t L>
112 : 0 : std::conditional_t<CanBeNull, _Bool<L>, Bool<L>> compile_cnf(ExprCompiler &C, const cnf::CNF &cnf)
113 : : {
114 : : using result_t = std::conditional_t<CanBeNull, _Bool<L>, Bool<L>>;
115 : :
116 [ # # # # : 0 : if (cnf.empty())
# # # # #
# # # ]
117 : 0 : return result_t(true);
118 : :
119 : 0 : std::optional<result_t> wasm_cnf, wasm_clause;
120 [ # # # # : 0 : for (auto &clause : cnf) {
# # # # #
# # # ]
121 : 0 : wasm_clause.reset();
122 [ # # # # : 0 : for (auto &pred : clause) {
# # # # #
# # # ]
123 : : /* Generate code for the literal of the predicate. */
124 [ # # # # : 0 : M_insist(pred.expr().type()->is_boolean());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
125 [ # # # # : 0 : auto compiled = M_CONSTEXPR_COND(CanBeNull, C.compile<_Bool<L>>(pred.expr()),
# # # # #
# # # # #
# # # # ]
126 : : C.compile<_Bool<L>>(pred.expr()).insist_not_null());
127 [ # # # # : 0 : auto wasm_pred = pred.negative() ? not compiled : compiled;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
128 : :
129 : : /* Add the predicate to the clause with an `or`. */
130 [ # # # # : 0 : if (wasm_clause)
# # # # #
# # # ]
131 [ # # # # : 0 : wasm_clause.emplace(*wasm_clause or wasm_pred);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
132 : : else
133 [ # # # # : 0 : wasm_clause.emplace(wasm_pred);
# # # # #
# # # ]
134 : 0 : }
135 [ # # # # : 0 : M_insist(bool(wasm_clause), "empty clause?");
# # # # #
# # # ]
136 : :
137 : : /* Add the clause to the CNF with an `and`. */
138 [ # # # # : 0 : if (wasm_cnf)
# # # # #
# # # ]
139 [ # # # # : 0 : wasm_cnf.emplace(*wasm_cnf and *wasm_clause);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
140 : : else
141 [ # # # # : 0 : wasm_cnf.emplace(*wasm_clause);
# # # # #
# # # ]
142 : : }
143 [ # # # # : 0 : M_insist(bool(wasm_cnf), "empty CNF?");
# # # # #
# # # ]
144 : :
145 [ # # # # : 0 : return *wasm_cnf;
# # # # #
# # # ]
146 : 0 : }
147 : :
148 : :
149 : : /*======================================================================================================================
150 : : * ExprCompiler
151 : : *====================================================================================================================*/
152 : :
153 : 0 : void ExprCompiler::operator()(const ast::ErrorExpr&) { M_unreachable("no errors at this stage"); }
154 : :
155 : 0 : void ExprCompiler::operator()(const ast::Designator &e)
156 : : {
157 [ # # ]: 0 : if (e.type()->is_none()) { // create NULL
158 [ # # # # : 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
# # # ]
159 : 0 : default: M_unreachable("invalid number of SIMD lanes");
160 [ # # ]: 0 : case 1: set(_I32x1::Null()); break;
161 [ # # # # ]: 0 : case 2: set(_I32x2::Null()); break;
162 [ # # ]: 0 : case 4: set(_I32x4::Null()); break;
163 [ # # ]: 0 : case 8: set(_I32x8::Null()); break;
164 [ # # ]: 0 : case 16: set(_I32x16::Null()); break;
165 [ # # ]: 0 : case 32: set(_I32x32::Null()); break;
166 : : }
167 : 0 : return;
168 : : }
169 : :
170 : : /* Search with fully qualified name. */
171 [ # # # # ]: 0 : Schema::Identifier id(e.table_name.text, e.attr_name.text.assert_not_none());
172 [ # # # # ]: 0 : set(env_.get(id));
173 : 0 : }
174 : :
175 : 0 : void ExprCompiler::operator()(const ast::Constant &e)
176 : : {
177 [ # # ]: 0 : if (e.type()->is_none()) { // create NULL
178 [ # # # # : 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
# # # ]
179 : 0 : default: M_unreachable("invalid number of SIMD lanes");
180 [ # # ]: 0 : case 1: set(_I32x1::Null()); break;
181 [ # # # # ]: 0 : case 2: set(_I32x2::Null()); break;
182 [ # # ]: 0 : case 4: set(_I32x4::Null()); break;
183 [ # # ]: 0 : case 8: set(_I32x8::Null()); break;
184 [ # # ]: 0 : case 16: set(_I32x16::Null()); break;
185 [ # # ]: 0 : case 32: set(_I32x32::Null()); break;
186 : : }
187 : 0 : return;
188 : : }
189 : :
190 : : /* Interpret constant. */
191 : 0 : auto value = Interpreter::eval(e);
192 : :
193 : 0 : auto set_constant = [this, &e, &value]<std::size_t L>(){
194 : : auto set_helper = overloaded {
195 : 0 : [this]<sql_type T>(T &&actual) { this->set(std::forward<T>(actual)); },
196 : 0 : [](auto&&) { M_unreachable("not a SQL type"); }
197 : : };
198 : :
199 : 0 : visit(overloaded {
200 [ # # # # : 0 : [&value, &set_helper](const Boolean&) { set_helper(_Bool<L>(value.as_b())); },
# # # # #
# # # ]
201 : 0 : [&value, &set_helper](const Numeric &n) {
202 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # ]
203 : : case Numeric::N_Int:
204 : : case Numeric::N_Decimal:
205 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
206 : : default:
207 : 0 : M_unreachable("invalid integer size");
208 : : case 8:
209 [ # # # # : 0 : set_helper(_I8<L>(value.as_i()));
# # # # #
# # # ]
210 : 0 : break;
211 : : case 16:
212 [ # # # # : 0 : set_helper(_I16<L>(value.as_i()));
# # # # #
# # # ]
213 : 0 : break;
214 : : case 32:
215 [ # # # # : 0 : set_helper(_I32<L>(value.as_i()));
# # # # #
# # # ]
216 : 0 : break;
217 : : case 64:
218 [ # # # # : 0 : set_helper(_I64<L>(value.as_i()));
# # # # #
# # # ]
219 : 0 : break;
220 : : }
221 : 0 : break;
222 : : case Numeric::N_Float:
223 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # ]
224 [ # # # # : 0 : set_helper(_Float<L>(value.as_f()));
# # # # #
# # # ]
225 : : else
226 [ # # # # : 0 : set_helper(_Double<L>(value.as_d()));
# # # # #
# # # ]
227 : 0 : }
228 : 0 : },
229 : 0 : [this, &value](const CharacterSequence&) {
230 : 0 : M_insist(L == 1, "string SIMDfication currently not supported");
231 [ # # # # : 0 : set(CodeGenContext::Get().get_literal_address(value.as<const char*>()));
# # # # #
# # # ]
232 : 0 : },
233 [ # # # # : 0 : [&value, &set_helper](const Date&) { set_helper(_I32<L>(value.as_i())); },
# # # # #
# # # ]
234 [ # # # # : 0 : [&value, &set_helper](const DateTime&) { set_helper(_I64<L>(value.as_i())); },
# # # # #
# # # ]
235 : 0 : [](const NoneType&) { M_unreachable("should've been handled earlier"); },
236 : 0 : [](auto&&) { M_unreachable("invalid type for given number of SIMD lanes"); },
237 : 0 : }, *e.type());
238 : 0 : };
239 [ # # # # : 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
# # # ]
240 : 0 : default: M_unreachable("invalid number of SIMD lanes");
241 : 0 : case 1: set_constant.operator()<1>(); break;
242 : 0 : case 2: set_constant.operator()<2>(); break;
243 : 0 : case 4: set_constant.operator()<4>(); break;
244 : 0 : case 8: set_constant.operator()<8>(); break;
245 : 0 : case 16: set_constant.operator()<16>(); break;
246 : 0 : case 32: set_constant.operator()<32>(); break;
247 : : }
248 : 0 : }
249 : :
250 : 0 : void ExprCompiler::operator()(const ast::UnaryExpr &e)
251 : : {
252 : : /* This is a helper to apply unary operations to `Expr<T>`s. It uses SFINAE within `overloaded` to only apply the
253 : : * operation if it is well typed, e.g. `+42` is ok whereas `+true` is not. */
254 : 0 : auto apply_unop = [this, &e](auto unop) {
255 : 0 : (*this)(*e.expr);
256 [ # # # # : 0 : std::visit(overloaded {
# # # # ]
257 : 0 : [](std::monostate&&) -> void { M_unreachable("illegal value"); },
258 : 0 : [this, &unop](auto &&expr) -> void requires requires { { unop(expr) } -> sql_type; } {
259 [ # # # # : 0 : this->set(unop(expr));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
260 : 0 : },
261 : 0 : [](auto &&expr) -> void requires (not requires { { unop(expr) } -> sql_type; }) {
262 : 0 : M_unreachable("illegal operation");
263 : : },
264 : 0 : }, get());
265 : 0 : };
266 : :
267 : : #define UNOP(OP) apply_unop(overloaded { \
268 : : [](auto &&expr) -> decltype(expr.operator OP()) { return expr.operator OP(); }, \
269 : : }); \
270 : : break
271 : :
272 [ # # # # : 0 : switch (e.op().type) {
# ]
273 : : default:
274 : 0 : M_unreachable("invalid operator");
275 : :
276 : 0 : case TK_PLUS: UNOP(+);
277 : 0 : case TK_MINUS: UNOP(-);
278 : 0 : case TK_TILDE: UNOP(~);
279 : 0 : case TK_Not: UNOP(not);
280 : : }
281 : : #undef UNOP
282 : 0 : }
283 : :
284 : 0 : void ExprCompiler::operator()(const ast::BinaryExpr &e)
285 : : {
286 : : /* This is a helper to apply binary operations to `Expr<T>`s. It uses SFINAE within `overloaded` to only apply the
287 : : * operation if it is well typed, e.g. `42 + 13` is ok whereas `true + 42` is not. */
288 : 0 : auto apply_binop = [this, &e](auto binop) {
289 : 0 : (*this)(*e.lhs);
290 : 0 : SQL_t lhs = get();
291 : :
292 [ # # # # : 0 : (*this)(*e.rhs);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
293 [ # # # # : 0 : SQL_t rhs = get();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
294 : :
295 [ # # # # : 0 : if (e.common_operand_type) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
296 [ # # # # : 0 : convert_in_place(lhs, e.common_operand_type); // convert in-place
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
297 [ # # # # : 0 : convert_in_place(rhs, e.common_operand_type); // convert in-place
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
298 : 0 : }
299 : :
300 [ # # # # : 0 : std::visit(overloaded {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
301 : : [](std::monostate&&) -> void { M_unreachable("illegal value"); },
302 : 0 : [this, &binop, &rhs](auto &&expr_lhs) -> void {
303 [ # # # # : 0 : std::visit(overloaded {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
304 : : [](std::monostate&&) -> void { M_unreachable("illegal value"); },
305 [ # # # # : 0 : [this, expr_lhs, &binop](auto &&expr_rhs) mutable -> void
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
306 : : requires requires { { binop(expr_lhs, expr_rhs) } -> sql_type; } {
307 [ # # # # : 0 : this->set(binop(expr_lhs, expr_rhs));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
308 : 0 : },
309 : 0 : [](auto &&expr_rhs) -> void
310 : : requires (not requires { { binop(expr_lhs, expr_rhs) } -> sql_type; }) {
311 : 0 : M_unreachable("illegal operation");
312 : : },
313 : 0 : }, rhs);
314 : 0 : },
315 : : }, lhs);
316 : 0 : };
317 : :
318 : : #define BINOP(OP) apply_binop( \
319 : : [](auto lhs, auto rhs) -> decltype(lhs.operator OP(rhs)) { return lhs.operator OP(rhs); } \
320 : : ); break
321 : : #define CMPOP(OP, STRCMP_OP) { \
322 : : if (e.lhs->type()->is_character_sequence()) { \
323 : : M_insist(e.rhs->type()->is_character_sequence()); \
324 : : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes"); \
325 : : apply_binop( \
326 : : [](NChar lhs, NChar rhs) -> _Boolx1 { \
327 : : return strcmp(lhs, rhs, STRCMP_OP); \
328 : : } \
329 : : ); break; \
330 : : } else { \
331 : : BINOP(OP); \
332 : : } \
333 : : }
334 : :
335 [ # # # # : 0 : switch (e.op().type) {
# # # # #
# # # # #
# ]
336 : : default:
337 : 0 : M_unreachable("illegal token type");
338 : :
339 : : /*----- Arithmetic operations --------------------------------------------------------------------------------*/
340 [ # # # # : 0 : case TK_PLUS: BINOP(+);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
341 [ # # # # : 0 : case TK_MINUS: BINOP(-);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
342 [ # # # # : 0 : case TK_ASTERISK: BINOP(*);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
343 [ # # # # : 0 : case TK_SLASH: BINOP(/);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
344 [ # # # # : 0 : case TK_PERCENT: BINOP(%);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
345 : :
346 : : /*----- Comparison operations --------------------------------------------------------------------------------*/
347 [ # # # # : 0 : case TK_EQUAL: CMPOP(==, EQ);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
348 [ # # # # : 0 : case TK_BANG_EQUAL: CMPOP(!=, NE);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
349 [ # # # # : 0 : case TK_LESS: CMPOP(<, LT);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
350 [ # # # # : 0 : case TK_LESS_EQUAL: CMPOP(<=, LE);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
351 [ # # # # : 0 : case TK_GREATER: CMPOP(>, GT);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
352 [ # # # # : 0 : case TK_GREATER_EQUAL: CMPOP(>=, GE);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
353 : :
354 : : /*----- CharacterSequence operations -------------------------------------------------------------------------*/
355 : : case TK_Like: {
356 : 0 : M_insist(e.lhs->type()->is_character_sequence());
357 : 0 : M_insist(e.rhs->type()->is_character_sequence());
358 : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
359 : 0 : (*this)(*e.lhs);
360 : 0 : NChar str = get<NChar>();
361 [ # # # # ]: 0 : if (auto static_pattern = cast<ast::Constant>(e.rhs.get())) { // check whether specialization is applicable
362 [ # # # # ]: 0 : auto pattern = Catalog::Get().pool(
363 [ # # # # : 0 : interpret(*static_pattern->tok.text.assert_not_none()) // interpret pattern to handle escaped chars
# # # # ]
364 : : );
365 [ # # # # : 0 : if (std::regex_match(*pattern, std::regex("%[^_%\\\\]+%"))) { // contains expression
# # # # ]
366 [ # # # # : 0 : set(like_contains(str, pattern));
# # ]
367 : 0 : break;
368 : : }
369 [ # # # # : 0 : if (std::regex_match(*pattern, std::regex("[^_%\\\\]+%"))) { // prefix expression
# # # # ]
370 [ # # # # : 0 : set(like_prefix(str, pattern));
# # ]
371 : 0 : break;
372 : : }
373 [ # # # # : 0 : if (std::regex_match(*pattern, std::regex("%[^_%\\\\]+"))) { // suffix expression
# # # # ]
374 [ # # # # : 0 : set(like_suffix(str, pattern));
# # ]
375 : 0 : break;
376 : : }
377 [ # # ]: 0 : }
378 : : /* no specialization applicable, fallback to general dynamic programming approach */
379 [ # # ]: 0 : (*this)(*e.rhs);
380 [ # # ]: 0 : NChar pattern = get<NChar>();
381 [ # # # # : 0 : set(like(str, pattern));
# # # # ]
382 : : break;
383 : 0 : }
384 : :
385 : : case TK_DOTDOT: {
386 : 0 : M_insist(e.lhs->type()->is_character_sequence());
387 : 0 : M_insist(e.rhs->type()->is_character_sequence());
388 : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
389 : 0 : (*this)(*e.lhs);
390 : 0 : NChar lhs = get<NChar>();
391 [ # # ]: 0 : (*this)(*e.rhs);
392 [ # # ]: 0 : NChar rhs = get<NChar>();
393 : :
394 [ # # # # : 0 : M_insist(e.lhs->can_be_null() == lhs.can_be_null());
# # ]
395 [ # # # # : 0 : M_insist(e.rhs->can_be_null() == rhs.can_be_null());
# # ]
396 : :
397 [ # # ]: 0 : Var<Ptr<Charx1>> res; // always set here
398 [ # # # # : 0 : bool res_can_be_null = lhs.can_be_null() or rhs.can_be_null();
# # ]
399 [ # # # # ]: 0 : std::size_t res_length = lhs.length() + rhs.length() + 1; // allocate space for terminating NUL byte
400 : :
401 [ # # ]: 0 : if (res_can_be_null) {
402 [ # # ]: 0 : auto [_ptr_lhs, is_nullptr_lhs] = lhs.split();
403 [ # # ]: 0 : auto [_ptr_rhs, is_nullptr_rhs] = rhs.split();
404 [ # # # # ]: 0 : Ptr<Charx1> ptr_lhs(_ptr_lhs), ptr_rhs(_ptr_rhs); // since structured bindings cannot be used in lambda capture
405 : :
406 [ # # # # : 0 : IF (is_nullptr_lhs or is_nullptr_rhs) {
# # ]
407 [ # # ]: 0 : res = Ptr<Charx1>::Nullptr();
408 [ # # ]: 0 : } ELSE {
409 [ # # ]: 0 : res = Module::Allocator().pre_malloc<char>(res_length); // create pre-allocation for result
410 [ # # # # : 0 : Var<Ptr<Charx1>> ptr(strncpy(res, ptr_lhs, U32x1(lhs.length()))); // since res must not be changed
# # # # ]
411 [ # # # # : 0 : strncpy(ptr, ptr_rhs, U32x1(rhs.size_in_bytes())).discard(); // copy with possible terminating NUL byte
# # # # #
# # # ]
412 [ # # ]: 0 : if (not rhs.guarantees_terminating_nul())
413 [ # # # # ]: 0 : *ptr = '\0'; // terminate with NUL byte
414 : 0 : };
415 : 0 : } else {
416 [ # # # # : 0 : res = Module::Allocator().pre_malloc<char>(res_length); // create pre-allocation for result
# # ]
417 [ # # # # : 0 : Var<Ptr<Charx1>> ptr(strncpy(res, lhs, U32x1(lhs.length()))); // since res must not be changed
# # # # #
# # # ]
418 [ # # # # : 0 : strncpy(ptr, rhs, U32x1(rhs.size_in_bytes())).discard(); // copy with possible terminating NUL byte
# # # # #
# # # ]
419 [ # # # # ]: 0 : if (not rhs.guarantees_terminating_nul())
420 [ # # # # ]: 0 : *ptr = '\0'; // terminate with NUL byte
421 : 0 : }
422 : :
423 [ # # # # : 0 : set(SQL_t(NChar(res, res_can_be_null, res_length, /* guarantees_terminating_nul= */ true)));
# # # # ]
424 : : break;
425 : 0 : }
426 : :
427 : : /*----- Logical operations -----------------------------------------------------------------------------------*/
428 : : case TK_And:
429 : : case TK_Or: {
430 : 0 : M_insist(e.lhs->type()->is_boolean());
431 : 0 : M_insist(e.rhs->type()->is_boolean());
432 : :
433 : 0 : (*this)(*e.lhs);
434 : 0 : _Boolx1 lhs = get<_Boolx1>();
435 [ # # ]: 0 : (*this)(*e.rhs);
436 [ # # ]: 0 : _Boolx1 rhs = get<_Boolx1>();
437 : :
438 [ # # # # ]: 0 : if (e.op().type == TK_And)
439 [ # # # # : 0 : set(lhs and rhs);
# # ]
440 : : else
441 [ # # # # : 0 : set(lhs or rhs);
# # ]
442 : :
443 : : break;
444 : 0 : }
445 : : }
446 : : #undef CMPOP
447 : : #undef BINOP
448 : 0 : }
449 : :
450 : 0 : void ExprCompiler::operator()(const ast::FnApplicationExpr &e)
451 : : {
452 [ # # # # : 0 : switch (e.get_function().fnid) {
# ]
453 : : default:
454 : 0 : M_unreachable("function kind not implemented");
455 : :
456 : : case m::Function::FN_UDF:
457 : 0 : M_unreachable("UDFs not yet supported");
458 : :
459 : : /*----- NULL check -------------------------------------------------------------------------------------------*/
460 : : case m::Function::FN_ISNULL: {
461 : 0 : (*this)(*e.args[0]);
462 : 0 : auto arg = get();
463 [ # # ]: 0 : std::visit(overloaded { // do not use constraint `is_sql_type` since `is_null()` returns a `PrimitiveExpr`
464 : 0 : [this]<sql_type T>(T actual) -> void requires requires { SQL_t(actual.is_null()); } {
465 [ # # # # : 0 : set(actual.is_null());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
466 : 0 : },
467 : 0 : []<sql_type T>(T actual) -> void requires (not requires { SQL_t(actual.is_null()); }) {
468 : 0 : M_unreachable("NULL check not supported");
469 : : },
470 : 0 : [](std::monostate) -> void { M_unreachable("invalid variant"); },
471 : : }, arg);
472 : : break;
473 : 0 : }
474 : :
475 : : /*----- Type cast --------------------------------------------------------------------------------------------*/
476 : : case m::Function::FN_INT: {
477 : 0 : (*this)(*e.args[0]);
478 : 0 : auto arg = get();
479 [ # # ]: 0 : convert_in_place<int32_t>(arg);
480 [ # # ]: 0 : set(std::move(arg));
481 : : break;
482 : 0 : }
483 : :
484 : : /*----- Aggregate functions ----------------------------------------------------------------------------------*/
485 : : case m::Function::FN_COUNT:
486 : : case m::Function::FN_MIN:
487 : : case m::Function::FN_MAX:
488 : : case m::Function::FN_SUM:
489 : : case m::Function::FN_AVG: {
490 : 0 : std::ostringstream oss;
491 [ # # ]: 0 : oss << e;
492 [ # # # # : 0 : Schema::Identifier id(Catalog::Get().pool(oss.str().c_str()));
# # # # ]
493 [ # # # # ]: 0 : set(env_.get(id));
494 : 0 : }
495 : 0 : }
496 : 0 : }
497 : :
498 : 0 : void ExprCompiler::operator()(const ast::QueryExpr &e)
499 : : {
500 : : /* Search with fully qualified name. */
501 [ # # # # : 0 : Schema::Identifier id(e.alias(), Catalog::Get().pool("$res"));
# # ]
502 [ # # # # ]: 0 : set(env_.get(id));
503 : 0 : }
504 : :
505 : 0 : SQL_boolean_t ExprCompiler::compile(const cnf::CNF &cnf)
506 : : {
507 [ # # # # ]: 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
508 : 0 : default: M_unreachable("invalid number of SIMD lanes");
509 [ # # # # : 0 : case 1: return cnf.can_be_null() ? compile_cnf<true, 1>(*this, cnf) : compile_cnf<false, 1>(*this, cnf);
# # # # #
# ]
510 [ # # # # : 0 : case 16: return cnf.can_be_null() ? compile_cnf<true, 16>(*this, cnf) : compile_cnf<false, 16>(*this, cnf);
# # # # #
# ]
511 [ # # # # : 0 : case 32: return cnf.can_be_null() ? compile_cnf<true, 32>(*this, cnf) : compile_cnf<false, 32>(*this, cnf);
# # # # #
# ]
512 : : }
513 : 0 : }
514 : :
515 : :
516 : :
517 : : /*======================================================================================================================
518 : : * Environment
519 : : *====================================================================================================================*/
520 : :
521 : : M_LCOV_EXCL_START
522 : : void Environment::dump(std::ostream &out) const
523 : : {
524 : : out << "WasmEnvironment\n` entries: { ";
525 : : for (auto it = exprs_.begin(), end = exprs_.end(); it != end; ++it) {
526 : : if (it != exprs_.begin()) out << ", ";
527 : : out << it->first;
528 : : }
529 : : out << " }" << std::endl;
530 : :
531 : : out << "WasmEnvironment\n` address entries: { ";
532 : : for (auto it = expr_addrs_.begin(), end = expr_addrs_.end(); it != end; ++it) {
533 : : if (it != expr_addrs_.begin()) out << ", ";
534 : : out << it->first;
535 : : }
536 : : out << " }" << std::endl;
537 : : }
538 : :
539 : : void Environment::dump() const { dump(std::cerr); }
540 : : M_LCOV_EXCL_STOP
541 : :
542 : :
543 : : /*======================================================================================================================
544 : : * CodeGenContext
545 : : *====================================================================================================================*/
546 : :
547 : : thread_local std::unique_ptr<CodeGenContext> CodeGenContext::the_context_;
548 : :
549 : :
550 : : /*======================================================================================================================
551 : : * compile data layout
552 : : *====================================================================================================================*/
553 : :
554 : : namespace m {
555 : :
556 : : namespace wasm {
557 : :
558 : : /** Compiles the data layout \p layout containing tuples of schema \p layout_schema such that it sequentially
559 : : * stores/loads (depending on \tparam IsStore) tuples of schema \p _tuple_value_schema starting at memory address \p
560 : : * base_address and tuple ID \p tuple_id. If \tparam SinglePass, the store has to be done in a single pass, i.e. the
561 : : * execution of the returned code must *not* be split among multiple function calls. Otherwise, the store does *not*
562 : : * have to be done in a single pass, i.e. the returned code may be emitted into a function which can be called
563 : : * multiple times and each call starts storing at exactly the point where it has ended in the last call. The given
564 : : * variable \p tuple_id will be incremented automatically before advancing to the next tuple (i.e. code for this will
565 : : * be emitted at the start of the block returned as third element). Predication is supported and emitted respectively
566 : : * for storing tuples. SIMDfication is supported and will be emitted iff \tparam L is greater than 1.
567 : : *
568 : : * Does not emit any code but returns three `wasm::Block`s containing code: the first one initializes all needed
569 : : * variables, the second one stores/loads one tuple, and the third one advances to the next tuple. Additionally, if not
570 : : * \tparam IsStore, adds the addresses of the values of tuples with schema \p _tuple_addr_schema into the current
571 : : * environment. */
572 : : template<bool IsStore, std::size_t L, bool SinglePass, bool PointerSharing, VariableKind Kind>
573 : : requires (L > 0) and (is_pow_2(L))
574 : : std::tuple<Block, Block, Block>
575 : 10 : compile_data_layout_sequential(const Schema &_tuple_value_schema, const Schema &_tuple_addr_schema,
576 : : Ptr<void> base_address, const storage::DataLayout &layout, const Schema &layout_schema,
577 : : Variable<uint32_t, Kind, false> &tuple_id)
578 : : {
579 [ # # # # : 10 : const auto tuple_value_schema = _tuple_value_schema.deduplicate().drop_constants();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
580 [ # # # # : 10 : const auto tuple_addr_schema = _tuple_addr_schema.deduplicate().drop_constants();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
581 : :
582 [ # # # # : 10 : M_insist(tuple_value_schema.num_entries() != 0, "sequential access must access at least one tuple schema entry");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
583 [ # # # # : 10 : M_insist(not IsStore or tuple_addr_schema.num_entries() == 0, "addresses are only computed for loads");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
584 : : #ifndef NDEBUG
585 [ # # # # : 70 : for (auto &e : tuple_value_schema)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
586 [ # # # # : 60 : M_insist(layout_schema.find(e.id) != layout_schema.cend(), "tuple value schema entry not found");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
587 [ + - # # : 11 : for (auto &e : tuple_addr_schema) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
588 [ # # # # : 0 : auto it = layout_schema.find(e.id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
589 [ + - # # : 1 : M_insist(it != layout_schema.cend(), "tuple address schema entry not found");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
590 [ + - # # : 1 : M_insist(not it->nullable(), "nullable tuple address schema entry not yet supported");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
591 [ # # # # : 0 : M_insist(not it->type->is_boolean(), "boolean tuple address schema entry not yet supported");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
592 [ # # # # : 0 : M_insist(not it->type->is_character_sequence(), "character sequence tuple address schema entry omitted");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
593 : : }
594 : : #endif
595 : :
596 : : /** code blocks for pointer/mask initialization, stores/loads of values, and stride jumps for pointers / updates
597 : : * of masks */
598 [ # # # # : 10 : Block inits("inits", false), stores("stores", false), loads("loads", false), jumps("jumps", false);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
599 : : ///> the values loaded for the entries in `tuple_value_schema`
600 [ # # # # : 10 : SQL_t values[tuple_value_schema.num_entries()];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
601 : : ///> the addresses for the entries in `tuple_addr_schema`
602 : : SQL_addr_t *addrs;
603 [ # # # # : 10 : if (not tuple_addr_schema.empty())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
604 : 0 : addrs = static_cast<SQL_addr_t*>(alloca(sizeof(SQL_addr_t) * tuple_addr_schema.num_entries()));
605 : : ///> the NULL information loaded for the entries in `tuple_value_schema`
606 : : Bool<L> *null_bits;
607 : : if constexpr (not IsStore)
608 : 10 : null_bits = static_cast<Bool<L>*>(alloca(sizeof(Bool<L>) * tuple_value_schema.num_entries()));
609 : :
610 : : using key_t = std::pair<uint8_t, uint64_t>;
611 : : ///> variable type for pointers dependent on whether access should be done using a single pass
612 : : using ptr_t = std::conditional_t<SinglePass, Var<Ptr<void>>, Global<Ptr<void>>>;
613 : : ///> variable type for masks dependent on whether access should be done using a single pass
614 : : using mask_t = std::conditional_t<SinglePass, Var<U32x1>, Global<U32x1>>;
615 : 0 : struct value_t
616 : : {
617 : : ptr_t ptr;
618 : : std::optional<mask_t> mask;
619 : : };
620 : : /** a map from bit offset (mod 8) and stride in bits to runtime pointer and mask; reset for each leaf;
621 : : * if \tparam PointerSharing, used to share pointers between attributes of the same leaf that have equal stride
622 : : * and to share masks between attributes of the same leaf that have equal offset (mod 8) */
623 : : std::conditional_t<
624 : : PointerSharing, std::unordered_map<key_t, value_t>, std::vector<std::pair<key_t, value_t>>
625 : 10 : > loading_context;
626 : :
627 [ # # # # : 10 : auto &env = CodeGenContext::Get().env(); // the current codegen environment
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
628 : :
629 : : if constexpr (L > 1) {
630 [ # # # # : 0 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
631 [ # # # # : 0 : Wasm_insist(tuple_id % uint32_t(L) == 0U, "must start at a tuple ID beginning a SIMD batch");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
632 : : }
633 : : }
634 : :
635 : : /*----- Check whether any of the entries in `tuple_value_schema` can be NULL, so that we need the NULL bitmap. -----*/
636 [ # # # # : 20 : const bool needs_null_bitmap = [&]() {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
637 [ # # # # : 10 : for (auto &tuple_entry : tuple_value_schema) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
638 [ # # # # : 10 : if (layout_schema[tuple_entry.id].second.nullable())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
639 : 10 : return true; // found an entry in `tuple_value_schema` that can be NULL according to `layout_schema`
640 : : }
641 : 0 : return false; // no attribute in `tuple_value_schema` can be NULL according to `layout_schema`
642 : 10 : }();
643 : 10 : bool has_null_bitmap = false; // indicates whether the data layout specifies a NULL bitmap
644 : :
645 : : /*----- If predication is used, introduce predication variable and update it before storing a tuple. -----*/
646 : 10 : const bool is_predicated = env.predicated();
647 [ # # # # : 10 : M_insist(not is_predicated or (IsStore and L == 1), "predication only supported for storing scalar tuples");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
648 : 10 : std::optional<Var<Boolx1>> pred;
649 [ # # # # : 10 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
650 [ # # # # : 0 : BLOCK_OPEN(stores) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
651 [ # # # # : 0 : pred = env.extract_predicate<_Boolx1>().is_true_and_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
652 : : }
653 : 0 : }
654 : :
655 : : /*----- Increment tuple ID before advancing to the next tuple pack. -----*/
656 : : if constexpr (IsStore) {
657 [ # # # # : 0 : BLOCK_OPEN(jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
658 [ # # # # : 0 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
659 [ # # # # : 0 : M_insist(L == 1);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
660 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
661 [ # # # # : 0 : tuple_id += pred->to<uint32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
662 : 0 : } else {
663 [ # # # # : 0 : tuple_id += uint32_t(L);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
664 : : }
665 : : }
666 : : } else {
667 [ + - # # : 20 : BLOCK_OPEN(jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
668 [ + - # # : 10 : tuple_id += uint32_t(L);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
669 : : }
670 : : }
671 : :
672 : : /*----- Visit the data layout. -----*/
673 [ # # # # : 10 : layout.for_sibling_leaves(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
674 [ # # # # : 22 : [&, &inits=inits, &jumps=jumps, &stores=stores, &loads=loads] // explicitly capture references non-const
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
675 : : (const std::vector<DataLayout::leaf_info_t> &leaves, const DataLayout::level_info_stack_t &levels,
676 : : uint64_t inode_offset_in_bits)
677 : : {
678 : : /*----- Clear the per-leaf data structure. -----*/
679 : 12 : loading_context.clear();
680 : :
681 : : /*----- Remember whether and where we found the NULL bitmap. -----*/
682 : 12 : std::optional<ptr_t> null_bitmap_ptr;
683 : 12 : std::optional<mask_t> null_bitmap_mask;
684 : : uint8_t null_bitmap_bit_offset;
685 : : uint64_t null_bitmap_stride_in_bits;
686 : :
687 : : /*----- Compute INode offset in bytes and INode iteration depending on the given tuple ID. -----*/
688 : 24 : auto compute_additional_inode_byte_offset = [&](U32x1 tuple_id) -> U64x1 {
689 : 40 : auto rec = [&](U32x1 curr_tuple_id, decltype(levels.cbegin()) curr, const decltype(levels.cend()) end,
690 : : auto rec) -> U64x1
691 : : {
692 [ # # # # : 28 : if (curr == end) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
693 [ # # # # : 12 : Wasm_insist(curr_tuple_id == tuple_id % uint32_t(levels.back().num_tuples));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
694 : 12 : return U64x1(0);
695 : : }
696 : :
697 [ # # # # : 16 : if (is_pow_2(curr->num_tuples)) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
698 [ # # # # : 10 : U32x1 child_iter = curr_tuple_id.clone() >> uint32_t(__builtin_ctzl(curr->num_tuples));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
699 [ # # # # : 10 : U32x1 inner_tuple_id = curr_tuple_id bitand uint32_t(curr->num_tuples - 1U);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
700 [ # # # # : 10 : M_insist(curr->stride_in_bits % 8 == 0, "INode stride must be byte aligned");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
701 [ # # # # : 10 : U64x1 offset_in_bytes = child_iter * uint64_t(curr->stride_in_bits / 8);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
702 [ # # # # : 10 : return offset_in_bytes + rec(inner_tuple_id, std::next(curr), end, rec);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
703 : 10 : } else {
704 [ # # # # : 6 : U32x1 child_iter = curr_tuple_id.clone() / uint32_t(curr->num_tuples);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
705 [ # # # # : 6 : U32x1 inner_tuple_id = curr_tuple_id % uint32_t(curr->num_tuples);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
706 [ # # # # : 6 : M_insist(curr->stride_in_bits % 8 == 0, "INode stride must be byte aligned");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
707 [ # # # # : 6 : U64x1 offset_in_bytes = child_iter * uint64_t(curr->stride_in_bits / 8);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
708 [ # # # # : 6 : return offset_in_bytes + rec(inner_tuple_id, std::next(curr), end, rec);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
709 : 6 : }
710 : 28 : };
711 [ # # # # : 12 : return rec(tuple_id.clone(), levels.cbegin(), levels.cend(), rec);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
712 : 0 : };
713 : 12 : std::optional<const Var<I32x1>> inode_byte_offset;
714 : 12 : std::optional<const Var<U32x1>> inode_iter;
715 [ # # # # : 24 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
716 [ # # # # : 12 : M_insist(inode_offset_in_bits % 8 == 0, "INode offset must be byte aligned");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
717 [ # # # # : 12 : inode_byte_offset.emplace(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
718 : 12 : int32_t(inode_offset_in_bits / 8)
719 [ # # # # : 12 : + compute_additional_inode_byte_offset(tuple_id).make_signed().template to<int32_t>()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
720 : : );
721 [ # # # # : 12 : M_insist(levels.back().num_tuples != 0, "INode must be large enough for at least one tuple");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
722 [ # # # # : 12 : if (levels.back().num_tuples != 1) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
723 [ # # # # : 8 : inode_iter.emplace(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
724 [ # # # # : 10 : is_pow_2(levels.back().num_tuples) ? tuple_id bitand uint32_t(levels.back().num_tuples - 1U)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + + + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
725 [ # # # # : 2 : : tuple_id % uint32_t(levels.back().num_tuples)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
726 : : );
727 : 8 : } else {
728 : : /* omit computation of INode iteration since it is always the first iteration, i.e. equals 0 */
729 : : }
730 : : };
731 : :
732 : : /*----- Iterate over sibling leaves, i.e. leaf children of a common parent INode, to emit code. -----*/
733 [ # # # # : 82 : for (auto &leaf_info : leaves) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
734 : 70 : const uint8_t bit_stride = leaf_info.stride_in_bits % 8; // need byte stride later for the stride jumps
735 : :
736 [ # # # # : 70 : if (leaf_info.leaf.index() == layout_schema.num_entries()) { // NULL bitmap
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
737 [ # # # # : 10 : if (not needs_null_bitmap)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
738 : 0 : continue;
739 : :
740 : : M_insist_no_ternary_logic();
741 [ # # # # : 10 : M_insist(not has_null_bitmap, "at most one bitmap may be specified");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
742 : 10 : has_null_bitmap = true;
743 [ # # # # : 10 : if (bit_stride) { // NULL bitmap with bit stride requires dynamic masking
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
744 [ # # # # : 4 : M_insist(L == 1, "SIMDfied loading of NULL bitmap with bit stride currently not supported");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
745 : :
746 [ # # # # : 4 : M_insist(bool(inode_iter), "stride requires repetition");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
747 [ # # # # : 4 : U64x1 leaf_offset_in_bits = leaf_info.offset_in_bits + *inode_iter * leaf_info.stride_in_bits;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
748 [ # # # # : 4 : U8x1 leaf_bit_offset = (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>() ; // mod 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
749 [ # # # # : 4 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
750 : :
751 : 4 : null_bitmap_bit_offset = leaf_info.offset_in_bits % 8;
752 : 4 : null_bitmap_stride_in_bits = leaf_info.stride_in_bits;
753 [ # # # # : 8 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
754 : : /*----- Initialize pointer and mask. -----*/
755 [ # # # # : 4 : null_bitmap_ptr.emplace(); // default-construct for globals to be able to use assignment below
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
756 [ # # # # : 4 : *null_bitmap_ptr = base_address.clone() + *inode_byte_offset + leaf_byte_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
757 [ # # # # : 4 : null_bitmap_mask.emplace(); // default-construct for globals to be able to use assignment below
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
758 [ # # # # : 4 : *null_bitmap_mask = 1U << leaf_bit_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
759 : : }
760 : :
761 : : /*----- Iterate over layout entries in *ascending* order. -----*/
762 : 4 : std::size_t prev_layout_idx = 0; ///< remember the bit offset of the previously accessed NULL bit
763 [ # # # # : 28 : for (std::size_t layout_idx = 0; layout_idx < layout_schema.num_entries(); ++layout_idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
764 [ # # # # : 24 : auto &layout_entry = layout_schema[layout_idx];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
765 [ # # # # : 24 : if (layout_entry.nullable()) { // layout entry may be NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
766 [ # # # # : 24 : auto tuple_it = tuple_value_schema.find(layout_entry.id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
767 [ # # # # : 24 : if (tuple_it == tuple_value_schema.end())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
768 : 0 : continue; // entry not contained in tuple schema
769 [ # # # # : 24 : M_insist(prev_layout_idx == 0 or layout_idx > prev_layout_idx,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
770 : : "layout entries not processed in ascending order");
771 [ # # # # : 24 : M_insist(*tuple_it->type == *layout_entry.type);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
772 : 24 : const auto delta = layout_idx - prev_layout_idx;
773 : 24 : const uint8_t bit_delta = delta % 8;
774 : 24 : const int32_t byte_delta = delta / 8;
775 : :
776 : 48 : auto advance_to_next_bit = [&]() {
777 [ # # # # : 24 : if (bit_delta) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
778 [ # # # # : 20 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
779 : 0 : M_insist(bool(pred));
780 [ # # # # : 0 : *null_bitmap_mask <<=
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
781 : 0 : Select(*pred, bit_delta, uint8_t(0)); // possibly advance mask
782 : 0 : } else {
783 : 20 : *null_bitmap_mask <<= bit_delta; // advance mask
784 : : }
785 : : /* If the mask surpasses the first byte, advance pointer to the next byte... */
786 [ # # # # : 20 : *null_bitmap_ptr += (*null_bitmap_mask bitand 0xffU).eqz().template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
787 : : /* ... and remove lowest byte from the mask. */
788 [ # # # # : 40 : *null_bitmap_mask = Select((*null_bitmap_mask bitand 0xffU).eqz(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
789 [ # # # # : 20 : *null_bitmap_mask >> 8U, *null_bitmap_mask);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
790 : 20 : }
791 [ # # # # : 24 : if (byte_delta) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
792 [ # # # # : 0 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
793 : 0 : M_insist(bool(pred));
794 [ # # # # : 0 : *null_bitmap_ptr +=
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
795 : 0 : Select(*pred, byte_delta, 0); // possibly advance pointer
796 : 0 : } else {
797 : 0 : *null_bitmap_ptr += byte_delta; // advance pointer
798 : : }
799 : 0 : }
800 : 24 : };
801 : :
802 : : if constexpr (IsStore) {
803 : : /*----- Store NULL bit depending on its type. -----*/
804 : 0 : auto store = [&]<typename T>() {
805 : 0 : BLOCK_OPEN(stores) {
806 [ # # # # : 0 : advance_to_next_bit();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
807 : :
808 [ # # # # : 0 : auto [value, is_null] = env.get<T>(tuple_it->id).split(); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
809 [ # # # # : 0 : value.discard(); // handled at entry leaf
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
810 [ # # # # : 0 : setbit(null_bitmap_ptr->template to<uint8_t*>(), is_null,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
811 [ # # # # : 0 : null_bitmap_mask->template to<uint8_t>()); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
812 : 0 : }
813 : 0 : };
814 [ # # # # : 0 : visit(overloaded{
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
815 : 0 : [&](const Boolean&) { store.template operator()<_Boolx1>(); },
816 : 0 : [&](const Numeric &n) {
817 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
818 : : case Numeric::N_Int:
819 : : case Numeric::N_Decimal:
820 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
821 : 0 : default: M_unreachable("invalid size");
822 : 0 : case 8: store.template operator()<_I8x1 >(); break;
823 : 0 : case 16: store.template operator()<_I16x1>(); break;
824 : 0 : case 32: store.template operator()<_I32x1>(); break;
825 : 0 : case 64: store.template operator()<_I64x1>(); break;
826 : : }
827 : 0 : break;
828 : : case Numeric::N_Float:
829 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
830 : 0 : store.template operator()<_Floatx1>();
831 : : else
832 : 0 : store.template operator()<_Doublex1>();
833 : 0 : }
834 : 0 : },
835 : 0 : [&](const CharacterSequence&) {
836 : 0 : BLOCK_OPEN(stores) {
837 [ # # # # : 0 : advance_to_next_bit();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
838 : :
839 [ # # # # : 0 : auto value = env.get<NChar>(tuple_it->id); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
840 [ # # # # : 0 : setbit(null_bitmap_ptr->template to<uint8_t*>(), value.is_null(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
841 [ # # # # : 0 : null_bitmap_mask->template to<uint8_t>()); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
842 : 0 : }
843 : 0 : },
844 : 0 : [&](const Date&) { store.template operator()<_I32x1>(); },
845 : 0 : [&](const DateTime&) { store.template operator()<_I64x1>(); },
846 : 0 : [](auto&&) { M_unreachable("invalid type"); },
847 : 0 : }, *tuple_it->type);
848 : : } else {
849 [ + - # # : 24 : const auto tuple_idx = std::distance(tuple_value_schema.begin(), tuple_it);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
850 [ + - # # : 48 : BLOCK_OPEN(loads) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
851 [ + - # # : 24 : advance_to_next_bit();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
852 : :
853 [ + - + - : 24 : U8x1 byte = *null_bitmap_ptr->template to<uint8_t*>(); // load the byte
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
854 [ + - # # : 24 : Var<Boolx1> value(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
855 [ + - + - : 24 : (byte bitand *null_bitmap_mask).template to<bool>()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
856 : : ); // mask bit with dynamic mask
857 [ + - # # : 24 : new (&null_bits[tuple_idx]) Boolx1(value);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
858 : : /* Address for NULL bits not yet supported. */
859 : 24 : }
860 : : }
861 : :
862 : 24 : prev_layout_idx = layout_idx;
863 : 24 : } else { // layout entry must not be NULL
864 : : #ifndef NDEBUG
865 : : if constexpr (IsStore) {
866 : : /*----- Check that value is also not NULL. -----*/
867 : : auto check = overloaded{
868 : 0 : [&]<sql_type T>() {
869 : 0 : BLOCK_OPEN(stores) {
870 [ # # # # : 0 : Wasm_insist(env.get<T>(layout_entry.id).not_null(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
871 : : "value of non-nullable entry must not be nullable");
872 : : }
873 : 0 : },
874 : 0 : []<typename>() {
875 : 0 : M_unreachable("invalid type for given number of SIMD lanes");
876 : : }
877 : : };
878 [ # # # # : 0 : visit(overloaded{
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
879 : 0 : [&](const Boolean&) { check.template operator()<_Bool<L>>(); },
880 : 0 : [&](const Numeric &n) {
881 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
882 : : case Numeric::N_Int:
883 : : case Numeric::N_Decimal:
884 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
885 : 0 : default: M_unreachable("invalid size");
886 : 0 : case 8: check.template operator()<_I8 <L>>(); break;
887 : 0 : case 16: check.template operator()<_I16<L>>(); break;
888 : 0 : case 32: check.template operator()<_I32<L>>(); break;
889 : 0 : case 64: check.template operator()<_I64<L>>(); break;
890 : : }
891 : 0 : break;
892 : : case Numeric::N_Float:
893 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
894 : 0 : check.template operator()<_Float<L>>();
895 : : else
896 : 0 : check.template operator()<_Double<L>>();
897 : 0 : }
898 : 0 : },
899 : 0 : [&](const CharacterSequence&) { check.template operator()<NChar>(); },
900 : 0 : [&](const Date&) { check.template operator()<_I32<L>>(); },
901 : 0 : [&](const DateTime&) { check.template operator()<_I64<L>>(); },
902 : 0 : [](auto&&) { M_unreachable("invalid type"); },
903 : 0 : }, *layout_entry.type);
904 : : }
905 : : #endif
906 : : }
907 : 24 : }
908 : :
909 : : /*----- Final advancement of the pointer and mask to match the leaf's stride. -----*/
910 : : /* This is done here (and not together with the other stride jumps further below) since we only need
911 : : * to advance by `delta` bits since we already have advanced by `prev_layout_idx` bits. */
912 : 4 : const auto delta = leaf_info.stride_in_bits - prev_layout_idx;
913 : 4 : const uint8_t bit_delta = delta % 8;
914 : 4 : const int32_t byte_delta = delta / 8;
915 [ # # # # : 4 : if (bit_delta) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
916 [ # # # # : 8 : BLOCK_OPEN(jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
917 [ # # # # : 4 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
918 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
919 [ # # # # : 0 : *null_bitmap_mask <<= Select(*pred, bit_delta, uint8_t(0)); // possibly advance mask
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
920 : 0 : } else {
921 [ # # # # : 4 : *null_bitmap_mask <<= bit_delta; // advance mask
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
922 : : }
923 : : /* If the mask surpasses the first byte, advance pointer to the next byte... */
924 [ # # # # : 4 : *null_bitmap_ptr += (*null_bitmap_mask bitand 0xffU).eqz().template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
925 : : /* ... and remove the lowest byte from the mask. */
926 [ # # # # : 8 : *null_bitmap_mask = Select((*null_bitmap_mask bitand 0xffU).eqz(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
927 [ # # # # : 4 : *null_bitmap_mask >> 8U, *null_bitmap_mask);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
928 : : }
929 : 4 : }
930 [ # # # # : 4 : if (byte_delta) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
931 [ # # # # : 0 : BLOCK_OPEN(jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
932 [ # # # # : 0 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
933 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
934 [ # # # # : 0 : *null_bitmap_ptr += Select(*pred, byte_delta, 0); // possibly advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
935 : 0 : } else {
936 [ # # # # : 0 : *null_bitmap_ptr += byte_delta; // advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
937 : : }
938 : : }
939 : 0 : }
940 : 4 : } else { // NULL bitmap without bit stride can benefit from static masking of NULL bits
941 [ # # # # : 6 : M_insist(L == 1 or L >= 16,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
942 : : "NULL bits must fill at least an entire SIMD vector when loading SIMDfied");
943 [ # # # # : 6 : M_insist(L == 1 or tuple_value_schema.num_entries() <= 64,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
944 : : "bytes containing a NULL bitmap must fit into scalar value when loading SIMDfied");
945 [ # # # # : 6 : M_insist(L == 1 or
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
946 : : std::max(ceil_to_pow_2(tuple_value_schema.num_entries()), 8UL) == leaf_info.stride_in_bits,
947 : : "NULL bitmaps must be packed s.t. the distance between two NULL bits of a single "
948 : : "attribute is a power of 2 when loading SIMDfied");
949 [ # # # # : 6 : M_insist(L == 1 or leaf_info.offset_in_bits % 8 == 0,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
950 : : "NULL bitmaps must not start with bit offset when loading SIMDfied");
951 : :
952 [ # # # # : 12 : auto byte_offset = [&]() -> I32x1 {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
953 [ # # # # : 6 : if (inode_iter and leaf_info.stride_in_bits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
954 : : /* omit `leaf_info.offset_in_bits` here to add it to the static offsets and masks;
955 : : * this is valid since no bit stride means that the leaf byte offset computation is
956 : : * independent of the static parts */
957 : 2 : U64x1 leaf_offset_in_bits = *inode_iter * leaf_info.stride_in_bits;
958 [ # # # # : 2 : U8x1 leaf_bit_offset = (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>(); // mod 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
959 [ # # # # : 2 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
960 [ # # # # : 4 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
961 [ # # # # : 2 : Wasm_insist(leaf_bit_offset == 0U, "no leaf bit offset without bit stride");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
962 : : }
963 [ # # # # : 2 : return *inode_byte_offset + leaf_byte_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
964 : 2 : } else {
965 : 4 : return *inode_byte_offset;
966 : : }
967 : 6 : }();
968 : :
969 : 6 : key_t key(leaf_info.offset_in_bits % 8, leaf_info.stride_in_bits);
970 : 14 : auto [it, inserted] =
971 [ # # # # : 12 : M_CONSTEXPR_COND(PointerSharing,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
972 : : loading_context.try_emplace(std::move(key)),
973 : : std::make_pair(loading_context.emplace(loading_context.end(), std::move(key), value_t()), true));
974 [ # # # # : 6 : if (inserted) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
975 [ # # # # : 4 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
976 [ # # # # : 2 : it->second.ptr = base_address.clone() + byte_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
977 : : }
978 : 2 : } else {
979 [ # # # # : 4 : byte_offset.discard();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
980 : : }
981 : 6 : const auto &ptr = it->second.ptr;
982 : :
983 : : ///> maps static byte offsets to already loaded bytes to reuse them; only needed for scalar loading
984 : 6 : std::unordered_map<int32_t, Var<U8x1>> loaded_bytes;
985 : :
986 : : using bytes_t = std::variant<std::monostate, Var<U8<L>>, Var<U16<L>>, Var<U32<L>>, Var<U64<L>>>;
987 : : ///> a SIMD vector containing all loaded NULL bitmaps; only needed for SIMDfied loading
988 : 6 : bytes_t bytes;
989 : : if constexpr (not IsStore and L > 1) {
990 : 0 : auto emplace = [&]<typename T>() {
991 : : using type = typename T::type;
992 : : static constexpr std::size_t lanes = T::num_simd_lanes;
993 : 0 : BLOCK_OPEN(loads) {
994 [ # # # # : 0 : bytes.template emplace<Var<T>>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
995 [ # # # # : 0 : *(ptr + leaf_info.offset_in_bits / 8).template to<type*, lanes>()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
996 : : );
997 : : }
998 : 0 : };
999 [ # # # # : 0 : switch (ceil_to_pow_2(tuple_value_schema.num_entries())) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1000 [ # # # # : 0 : default: emplace.template operator()<U8 <L>>(); break; // <= 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1001 [ # # # # : 0 : case 16: emplace.template operator()<U16<L>>(); break;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1002 [ # # # # : 0 : case 32: emplace.template operator()<U32<L>>(); break;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1003 [ # # # # : 0 : case 64: emplace.template operator()<U64<L>>(); break;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1004 : : }
1005 : : }
1006 : :
1007 : : /*----- For each tuple entry that can be NULL, create a store/load with static offset and mask. --*/
1008 [ # # # # : 42 : for (std::size_t tuple_idx = 0; tuple_idx != tuple_value_schema.num_entries(); ++tuple_idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1009 [ # # # # : 36 : auto &tuple_entry = tuple_value_schema[tuple_idx];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1010 [ # # # # : 144 : const auto &[layout_idx, layout_entry] = layout_schema[tuple_entry.id];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1011 [ # # # # : 72 : M_insist(*tuple_entry.type == *layout_entry.type);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1012 [ # # # # : 36 : if (layout_entry.nullable()) { // layout entry may be NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1013 : 72 : const uint8_t static_bit_offset = (leaf_info.offset_in_bits + layout_idx) % 8;
1014 : 72 : const int32_t static_byte_offset = (leaf_info.offset_in_bits + layout_idx) / 8;
1015 : : if constexpr (IsStore) {
1016 : : /*----- Store NULL bit depending on its type. -----*/
1017 : : auto store = overloaded{
1018 : 0 : [&]<sql_type T>() {
1019 : 0 : BLOCK_OPEN(stores) {
1020 [ # # # # : 0 : auto [value, is_null] = env.get<T>(tuple_entry.id).split(); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1021 [ # # # # : 0 : value.discard(); // handled at entry leaf
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1022 : : if constexpr (L == 1) {
1023 : : Ptr<U8x1> byte_ptr =
1024 [ # # # # : 0 : (ptr + static_byte_offset).template to<uint8_t*>(); // compute byte address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1025 [ # # # # : 0 : setbit<U8x1>(byte_ptr, is_null, static_bit_offset); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1026 : 0 : } else {
1027 [ # # # # : 0 : auto store = [&, is_null, layout_idx]<typename U>() { // copy due to structured binding
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1028 : : using type = typename U::type;
1029 : : static constexpr std::size_t lanes = U::num_simd_lanes;
1030 : : Ptr<U> bytes_ptr =
1031 [ # # # # : 0 : (ptr + leaf_info.offset_in_bits / 8).template to<type*, lanes>(); // compute bytes address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1032 [ # # # # : 0 : setbit<U>(bytes_ptr, is_null, layout_idx); // update bits
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1033 : 0 : };
1034 [ # # # # : 0 : switch (ceil_to_pow_2(tuple_value_schema.num_entries())) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1035 [ # # # # : 0 : default: store.template operator()<U8 <L>>(); break; // <= 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1036 [ # # # # : 0 : case 16: store.template operator()<U16<L>>(); break;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1037 [ # # # # : 0 : case 32: store.template operator()<U32<L>>(); break;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1038 [ # # # # : 0 : case 64: store.template operator()<U64<L>>(); break;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1039 : : }
1040 : 0 : }
1041 : 0 : }
1042 : 0 : },
1043 : 0 : []<typename>() {
1044 : 0 : M_unreachable("invalid type for given number of SIMD lanes");
1045 : : }
1046 : : };
1047 [ # # # # : 0 : visit(overloaded{
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1048 : 0 : [&](const Boolean&) { store.template operator()<_Bool<L>>(); },
1049 : 0 : [&](const Numeric &n) {
1050 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1051 : : case Numeric::N_Int:
1052 : : case Numeric::N_Decimal:
1053 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1054 : 0 : default: M_unreachable("invalid size");
1055 : 0 : case 8: store.template operator()<_I8 <L>>(); break;
1056 : 0 : case 16: store.template operator()<_I16<L>>(); break;
1057 : 0 : case 32: store.template operator()<_I32<L>>(); break;
1058 : 0 : case 64: store.template operator()<_I64<L>>(); break;
1059 : : }
1060 : 0 : break;
1061 : : case Numeric::N_Float:
1062 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1063 : 0 : store.template operator()<_Float<L>>();
1064 : : else
1065 : 0 : store.template operator()<_Double<L>>();
1066 : 0 : }
1067 : 0 : },
1068 : 0 : [&](const CharacterSequence&) {
1069 : 0 : M_insist(L == 1, "string SIMDfication currently not supported");
1070 : 0 : BLOCK_OPEN(stores) {
1071 [ # # # # : 0 : auto value = env.get<NChar>(tuple_entry.id); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1072 : : Ptr<U8x1> byte_ptr =
1073 [ # # # # : 0 : (ptr + static_byte_offset).template to<uint8_t*>(); // compute byte address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1074 [ # # # # : 0 : setbit<U8x1>(byte_ptr, value.is_null(), static_bit_offset); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1075 : 0 : }
1076 : 0 : },
1077 : 0 : [&](const Date&) { store.template operator()<_I32<L>>(); },
1078 : 0 : [&](const DateTime&) { store.template operator()<_I64<L>>(); },
1079 : 0 : [](auto&&) { M_unreachable("invalid type"); },
1080 : 0 : }, *tuple_entry.type);
1081 : : } else {
1082 : : /*----- Load NULL bit. -----*/
1083 [ + - # # : 72 : BLOCK_OPEN(loads) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1084 : : if constexpr (L == 1) {
1085 [ + - # # : 48 : auto [it, inserted] = loaded_bytes.try_emplace(static_byte_offset);
# # # # #
# # # ]
1086 [ + + # # : 36 : if (inserted)
# # # # #
# # # ]
1087 [ + - + - : 6 : it->second = *(ptr + static_byte_offset).template to<uint8_t*>(); // load the byte
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1088 : 36 : const auto &byte = it->second;
1089 : 36 : const uint8_t static_mask = 1U << static_bit_offset;
1090 [ + - + - : 36 : Var<Boolx1> value((byte bitand static_mask).to<bool>()); // mask bit with static mask
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1091 [ + - # # : 36 : new (&null_bits[tuple_idx]) Boolx1(value);
# # # # #
# # # ]
1092 : : /* Address for NULL bits not yet supported. */
1093 : 36 : } else {
1094 [ # # # # : 0 : std::visit(overloaded{
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1095 : 0 : [&, layout_idx]<typename T> // copy due to structured binding
1096 : : (const Variable<T, VariableKind::Local, false, L> &_bytes)
1097 : : requires (L >= 16) {
1098 : 0 : PrimitiveExpr<T, L> static_mask(1U << layout_idx);
1099 [ # # # # : 0 : Var<Bool<L>> value(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1100 [ # # # # : 0 : (_bytes bitand static_mask).template to<bool>() // mask bits with static mask
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1101 : : );
1102 [ # # # # : 0 : new (&null_bits[tuple_idx]) Bool<L>(value);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1103 : : /* Address for NULL bits not yet supported. */
1104 : 0 : },
1105 : 0 : [](auto&) { M_unreachable("invalid number of SIMD lanes"); },
1106 : : [](std::monostate&) { M_unreachable("invalid variant"); },
1107 : : }, const_cast<const bytes_t&>(bytes));
1108 : : }
1109 : : }
1110 : : }
1111 : 36 : } else { // entry must not be NULL
1112 : : #ifndef NDEBUG
1113 : : if constexpr (IsStore) {
1114 : : /*----- Check that value is also not NULL. -----*/
1115 : 0 : auto check = overloaded{
1116 [ # # # # : 0 : [&, layout_entry]<sql_type T>() { // copy due to structured binding
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1117 : 0 : BLOCK_OPEN(stores) {
1118 [ # # # # : 0 : Wasm_insist(env.get<T>(layout_entry.id).not_null(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1119 : : "value of non-nullable entry must not be nullable");
1120 : : }
1121 : 0 : },
1122 : 0 : []<typename>() {
1123 : 0 : M_unreachable("invalid type for given number of SIMD lanes");
1124 : : }
1125 : : };
1126 [ # # # # : 0 : visit(overloaded{
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1127 : 0 : [&](const Boolean&) { check.template operator()<_Bool<L>>(); },
1128 : 0 : [&](const Numeric &n) {
1129 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1130 : : case Numeric::N_Int:
1131 : : case Numeric::N_Decimal:
1132 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1133 : 0 : default: M_unreachable("invalid size");
1134 : 0 : case 8: check.template operator()<_I8 <L>>(); break;
1135 : 0 : case 16: check.template operator()<_I16<L>>(); break;
1136 : 0 : case 32: check.template operator()<_I32<L>>(); break;
1137 : 0 : case 64: check.template operator()<_I64<L>>(); break;
1138 : : }
1139 : 0 : break;
1140 : : case Numeric::N_Float:
1141 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1142 : 0 : check.template operator()<_Float<L>>();
1143 : : else
1144 : 0 : check.template operator()<_Double<L>>();
1145 : 0 : }
1146 : 0 : },
1147 : 0 : [&](const CharacterSequence&) { check.template operator()<NChar>(); },
1148 : 0 : [&](const Date&) { check.template operator()<_I32<L>>(); },
1149 : 0 : [&](const DateTime&) { check.template operator()<_I64<L>>(); },
1150 : 0 : [](auto&&) { M_unreachable("invalid type"); },
1151 : 0 : }, *tuple_entry.type);
1152 : 0 : }
1153 : : #endif
1154 : : }
1155 : 36 : }
1156 : 6 : }
1157 : 10 : } else { // regular entry
1158 [ # # # # : 60 : auto &layout_entry = layout_schema[leaf_info.leaf.index()];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1159 [ # # # # : 60 : M_insist(*layout_entry.type == *leaf_info.leaf.type());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1160 [ # # # # : 60 : auto tuple_value_it = tuple_value_schema.find(layout_entry.id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1161 [ # # # # : 60 : auto tuple_addr_it = tuple_addr_schema.find(layout_entry.id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1162 [ # # # # : 60 : if (tuple_value_it == tuple_value_schema.end() and tuple_addr_it == tuple_addr_schema.end())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1163 : 0 : continue; // entry not contained in both tuple schemas
1164 [ # # # # : 60 : auto tuple_it = tuple_value_it != tuple_value_schema.end() ? tuple_value_it : tuple_addr_it;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1165 [ # # # # : 60 : M_insist(*tuple_it->type == *layout_entry.type);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1166 [ # # # # : 60 : const auto tuple_value_idx = std::distance(tuple_value_schema.begin(), tuple_value_it);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1167 [ # # # # : 60 : const auto tuple_addr_idx = std::distance(tuple_addr_schema.begin(), tuple_addr_it);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1168 : :
1169 [ # # # # : 60 : if (bit_stride) { // entry with bit stride requires dynamic masking (for scalar loading)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1170 [ # # # # : 12 : M_insist(tuple_it->type->is_boolean(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1171 : : "leaf bit stride currently only for `Boolean` supported");
1172 [ # # # # : 12 : M_insist(L == 1 or L >= 16,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1173 : : "booleans must fill at least an entire SIMD vector when loading SIMDfied");
1174 [ # # # # : 12 : M_insist(L <= 64, "bytes containing booleans must fit into scalar value when loading SIMDfied");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1175 [ # # # # : 12 : M_insist(L == 1 or leaf_info.stride_in_bits == 1,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1176 : : "booleans must be packed consecutively when loading SIMDfied");
1177 : :
1178 [ # # # # : 12 : M_insist(bool(inode_iter), "stride requires repetition");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1179 [ # # # # : 12 : U64x1 leaf_offset_in_bits = leaf_info.offset_in_bits + *inode_iter * leaf_info.stride_in_bits;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1180 [ # # # # : 12 : U8x1 leaf_bit_offset = (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>() ; // mod 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1181 [ # # # # : 12 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1182 : :
1183 : : if constexpr (L > 1) {
1184 [ # # # # : 0 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1185 [ # # # # : 0 : Wasm_insist(leaf_bit_offset == 0U,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1186 : : "booleans must not start with bit offset when loading SIMDfied");
1187 : : }
1188 : : }
1189 : :
1190 : 12 : key_t key(leaf_info.offset_in_bits % 8, leaf_info.stride_in_bits);
1191 : 68 : auto [it, inserted] =
1192 [ # # # # : 24 : M_CONSTEXPR_COND(PointerSharing,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1193 : : loading_context.try_emplace(std::move(key)),
1194 : : std::make_pair(loading_context.emplace(loading_context.end(), std::move(key), value_t()), true));
1195 [ # # # # : 24 : M_insist(inserted == not it->second.mask);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1196 [ # # # # : 12 : if (inserted) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1197 [ # # # # : 20 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1198 : : /* do not add `leaf_byte_offset` to pointer here as it may be different for shared entries */
1199 [ # # # # : 10 : it->second.ptr = base_address.clone() + *inode_byte_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1200 [ # # # # : 10 : it->second.mask.emplace(); // default-construct for globals to be able to use assignment below
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1201 : : if constexpr (L == 1)
1202 [ # # # # : 10 : *it->second.mask = 1U << leaf_bit_offset; // init mask for scalar loading
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # ]
1203 : : /* no dynamic mask required for SIMDfied loading */
1204 : : }
1205 : 10 : } else {
1206 [ # # # # : 2 : leaf_bit_offset.discard();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1207 : : }
1208 : 12 : const auto &ptr = it->second.ptr;
1209 : :
1210 : : if constexpr (IsStore) {
1211 : : if constexpr (sql_type<_Bool<L>>) {
1212 : : /*----- Store value. -----*/
1213 [ # # # # : 0 : BLOCK_OPEN(stores) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1214 [ # # # # : 0 : auto [value, is_null] = env.get<_Bool<L>>(tuple_it->id).split(); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1215 [ # # # # : 0 : is_null.discard(); // handled at NULL bitmap leaf
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1216 : : if constexpr (L == 1) {
1217 : : Ptr<U8x1> byte_ptr =
1218 [ # # # # : 0 : (ptr + leaf_byte_offset).template to<uint8_t*>(); // compute byte address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1219 : 0 : const auto &mask = *it->second.mask;
1220 [ # # # # : 0 : setbit(byte_ptr, value, mask.template to<uint8_t>()); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1221 : 0 : } else {
1222 : : using bytes_t = uint_t<L / 8>;
1223 : : Ptr<PrimitiveExpr<bytes_t>> bytes_ptr =
1224 [ # # # # : 0 : (ptr + leaf_byte_offset).template to<bytes_t*>(); // compute bytes address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1225 [ # # # # : 0 : *bytes_ptr = value.bitmask().template to<bytes_t>(); // update all bits at once
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1226 : 0 : }
1227 : 0 : }
1228 : : } else {
1229 [ # # # # : 0 : M_unreachable("invalid type for given number of SIMD lanes");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1230 : : }
1231 : : } else {
1232 : : if constexpr (sql_type<_Bool<L>>) {
1233 : : /*----- Load value. -----*/
1234 [ + - # # : 24 : BLOCK_OPEN(loads) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1235 : : if constexpr (L == 1) {
1236 [ + - + - : 12 : U8x1 byte = *(ptr + leaf_byte_offset).template to<uint8_t*>(); // load byte
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1237 : 12 : const auto &mask = *it->second.mask;
1238 [ + - # # : 12 : if (tuple_value_it != tuple_value_schema.end()) {
# # # # #
# # # ]
1239 [ + - # # : 12 : Var<Boolx1> value(
# # # # #
# # # ]
1240 [ + - + - : 12 : (byte.clone() bitand mask.template to<uint8_t>()).template to<bool>() // mask bit with dynamic mask
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1241 : : );
1242 [ + - + - : 12 : new (&values[tuple_value_idx]) SQL_t(_Boolx1(value));
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1243 : 12 : }
1244 : : /* Address for booleans not yet supported. */
1245 [ + - # # : 12 : byte.discard();
# # # # #
# # # ]
1246 : 12 : } else {
1247 : : using bytes_t = uint_t<L / 8>;
1248 [ # # # # : 0 : const Var<PrimitiveExpr<bytes_t>> bytes(
# # # # #
# # # # #
# # # # #
# # # #
# ]
1249 [ # # # # : 0 : *(ptr + leaf_byte_offset).template to<bytes_t*>() // load bytes
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1250 : : ); // create local variable to avoid cloning when broadcasting `bytes`
1251 : 0 : auto create_mask = [&]<std::size_t... Is>(std::index_sequence<Is...>) {
1252 : 0 : return PrimitiveExpr<bytes_t, L>(bytes_t(1UL << Is)...);
1253 : : };
1254 [ # # # # : 0 : auto static_mask = create_mask(std::make_index_sequence<L>());
# # # # #
# # # # #
# # # # #
# # # #
# ]
1255 [ # # # # : 0 : if (tuple_value_it != tuple_value_schema.end()) {
# # # # #
# # # # #
# # # # #
# # # #
# ]
1256 [ # # # # : 0 : Var<Bool<L>> value(
# # # # #
# # # # #
# # # # #
# # # #
# ]
1257 [ # # # # : 0 : (bytes.template broadcast<L>() bitand static_mask).template to<bool>() // mask bits with static mask
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1258 : : );
1259 [ # # # # : 0 : new (&values[tuple_value_idx]) SQL_t(_Bool<L>(value));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1260 : 0 : } else {
1261 [ # # # # : 0 : bytes.val().discard(); // XXX: remove once address for booleans are supported
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1262 : : }
1263 : : /* Address for booleans not yet supported. */
1264 : 0 : }
1265 : : }
1266 : : } else {
1267 [ # # # # : 0 : M_unreachable("invalid type for given number of SIMD lanes");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1268 : : }
1269 : : }
1270 : 12 : } else { // entry without bit stride; if masking is required, we can use a static mask
1271 [ # # # # : 96 : auto byte_offset = [&]() -> I32x1 {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1272 [ # # # # : 48 : if (inode_iter and leaf_info.stride_in_bits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1273 : : /* omit `leaf_info.offset_in_bits` here to use it as static offset and mask;
1274 : : * this is valid since no bit stride means that the leaf byte offset computation is
1275 : : * independent of the static parts */
1276 : 24 : U64x1 leaf_offset_in_bits = *inode_iter * leaf_info.stride_in_bits;
1277 [ # # # # : 24 : U8x1 leaf_bit_offset = (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>(); // mod 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1278 [ # # # # : 24 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1279 [ # # # # : 48 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1280 [ # # # # : 24 : Wasm_insist(leaf_bit_offset == 0U, "no leaf bit offset without bit stride");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1281 : : }
1282 [ # # # # : 24 : return *inode_byte_offset + leaf_byte_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1283 : 24 : } else {
1284 : 24 : return *inode_byte_offset;
1285 : : }
1286 : 48 : }();
1287 : :
1288 : 48 : const uint8_t static_bit_offset = leaf_info.offset_in_bits % 8;
1289 : 48 : const int32_t static_byte_offset = leaf_info.offset_in_bits / 8;
1290 : :
1291 : 48 : key_t key(leaf_info.offset_in_bits % 8, leaf_info.stride_in_bits);
1292 : 128 : auto [it, inserted] =
1293 [ # # # # : 96 : M_CONSTEXPR_COND(PointerSharing,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1294 : : loading_context.try_emplace(std::move(key)),
1295 : : std::make_pair(loading_context.emplace(loading_context.end(), std::move(key), value_t()), true));
1296 [ # # # # : 48 : if (inserted) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1297 [ # # # # : 64 : BLOCK_OPEN(inits) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1298 [ # # # # : 32 : it->second.ptr = base_address.clone() + byte_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1299 : : }
1300 : 32 : } else {
1301 [ # # # # : 16 : byte_offset.discard();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1302 : : }
1303 : 48 : const auto &ptr = it->second.ptr;
1304 : :
1305 : : /*----- Store value depending on its type. -----*/
1306 : : auto store = overloaded{
1307 : 48 : [&]<sql_type T>() {
1308 : : using type = typename T::type;
1309 : : static constexpr std::size_t lanes = T::num_simd_lanes;
1310 : 0 : M_insist(static_bit_offset == 0,
1311 : : "leaf offset of `Numeric`, `Date`, or `DateTime` must be byte aligned");
1312 : 0 : BLOCK_OPEN(stores) {
1313 [ # # # # : 0 : auto [value, is_null] = env.get<T>(tuple_it->id).split(); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1314 [ # # # # : 0 : is_null.discard(); // handled at NULL bitmap leaf
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1315 [ # # # # : 0 : *(ptr + static_byte_offset).template to<type*, lanes>() = value;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1316 : 0 : }
1317 : 0 : },
1318 : 0 : []<typename>() {
1319 : 0 : M_unreachable("invalid type for given number of SIMD lanes");
1320 : : }
1321 : : };
1322 : : /*----- Load value depending on its type. -----*/
1323 : : auto load = overloaded{
1324 : 78 : [&]<sql_type T>() {
1325 : : using type = typename T::type;
1326 : : static constexpr std::size_t lanes = T::num_simd_lanes;
1327 : 30 : M_insist(static_bit_offset == 0,
1328 : : "leaf offset of `Numeric`, `Date`, or `DateTime` must be byte aligned");
1329 : 60 : BLOCK_OPEN(loads) {
1330 [ - + # # : 30 : if (tuple_value_it != tuple_value_schema.end()) {
# # - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1331 [ + - # # : 30 : Var<PrimitiveExpr<type, lanes>> value(
# # + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1332 [ + - + - : 30 : *(ptr + static_byte_offset).template to<type*, lanes>()
+ - # # #
# # # # #
# # # # +
- + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1333 : : );
1334 [ + - + - : 30 : new (&values[tuple_value_idx]) SQL_t(T(value));
- + # # #
# # # # #
# # # # +
- + - - +
+ - + - -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1335 : 30 : }
1336 [ - + # # : 30 : if (tuple_addr_it != tuple_addr_schema.end())
# # - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1337 [ # # # # : 0 : new (&addrs[tuple_addr_idx]) SQL_addr_t(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1338 [ # # # # : 0 : (ptr + static_byte_offset).template to<type*, lanes>()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1339 : : );
1340 : : }
1341 : 30 : },
1342 : 0 : []<typename>() {
1343 : 0 : M_unreachable("invalid type for given number of SIMD lanes");
1344 : : }
1345 : : };
1346 : : /*----- Select call target (store or load) and visit attribute type. -----*/
1347 : : #define CALL(TYPE) if constexpr (IsStore) store.template operator()<TYPE>(); else load.template operator()<TYPE>()
1348 [ # # # # : 240 : visit(overloaded{
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1349 : 56 : [&](const Boolean&) {
1350 : 8 : M_insist(L == 1 or leaf_info.stride_in_bits == 8,
1351 : : "booleans must be packed consecutively in bytes when loading SIMDfied");
1352 : : if constexpr (sql_type<_Bool<L>>) {
1353 : : if constexpr (IsStore) {
1354 : : /*----- Store value. -----*/
1355 : 0 : BLOCK_OPEN(stores) {
1356 [ # # # # : 0 : auto [value, is_null] = env.get<_Bool<L>>(tuple_it->id).split(); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1357 [ # # # # : 0 : is_null.discard(); // handled at NULL bitmap leaf
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1358 : : Ptr<U8<L>> byte_ptr =
1359 [ # # # # : 0 : (ptr + static_byte_offset).template to<uint8_t*, L>(); // compute byte address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1360 [ # # # # : 0 : setbit<U8<L>>(byte_ptr, value, static_bit_offset); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1361 : 0 : }
1362 : : } else {
1363 : : /*----- Load value. -----*/
1364 : 16 : BLOCK_OPEN(loads) {
1365 : : U8<L> byte =
1366 [ + - + - : 8 : *(ptr + static_byte_offset).template to<uint8_t*, L>(); // load byte
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1367 [ + - # # : 8 : U8<L> static_mask(1U << static_bit_offset);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1368 : :
1369 [ + - # # : 8 : if (tuple_value_it != tuple_value_schema.end()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1370 [ + - # # : 8 : Var<Bool<L>> value(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1371 [ + - + - : 8 : (byte.clone() bitand static_mask.clone()).template to<bool>() // mask bit with static mask
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1372 : : );
1373 [ + - + - : 8 : new (&values[tuple_value_idx]) SQL_t(_Bool<L>(value));
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1374 : 8 : }
1375 : : /* Address for booleans not yet supported. */
1376 [ + - # # : 8 : byte.discard();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1377 [ + - # # : 8 : static_mask.discard();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1378 : 8 : }
1379 : : }
1380 : : } else {
1381 : 0 : M_unreachable("invalid type for given number of SIMD lanes");
1382 : : }
1383 : 8 : },
1384 : 78 : [&](const Numeric &n) {
1385 [ # # # # : 30 : switch (n.kind) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1386 : : case Numeric::N_Int:
1387 : : case Numeric::N_Decimal:
1388 [ # # # # : 20 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# - + - -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1389 : 0 : default: M_unreachable("invalid size");
1390 : 10 : case 8: CALL(_I8 <L>); break;
1391 : 0 : case 16: CALL(_I16<L>); break;
1392 : 0 : case 32: CALL(_I32<L>); break;
1393 : 10 : case 64: CALL(_I64<L>); break;
1394 : : }
1395 : 20 : break;
1396 : : case Numeric::N_Float:
1397 [ # # # # : 10 : if (n.size() <= 32)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1398 : 10 : CALL(_Float<L>);
1399 : : else
1400 : 0 : CALL(_Double<L>);
1401 : 10 : }
1402 : 30 : },
1403 : 58 : [&](const CharacterSequence &cs) {
1404 : 10 : M_insist(L == 1, "string SIMDfication currently not supported");
1405 : 10 : M_insist(static_bit_offset == 0, "leaf offset of `CharacterSequence` must be byte aligned");
1406 : : if constexpr (IsStore) {
1407 : : /*----- Store value. -----*/
1408 : 0 : BLOCK_OPEN(stores) {
1409 [ # # # # : 0 : auto value = env.get<NChar>(tuple_it->id); // get value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1410 [ # # # # : 0 : IF (value.clone().not_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1411 [ # # # # : 0 : Ptr<Charx1> address((ptr + static_byte_offset).template to<char*>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1412 [ # # # # : 0 : strncpy(address, value, U32x1(cs.size() / 8)).discard();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1413 : 0 : };
1414 : 0 : }
1415 : : } else {
1416 : : /*----- Load value. -----*/
1417 : 20 : BLOCK_OPEN(loads) {
1418 [ + - + - : 10 : Ptr<Charx1> address((ptr + static_byte_offset).template to<char*>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1419 [ - + # # : 20 : new (&values[tuple_value_idx]) SQL_t(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1420 [ + - + - : 10 : NChar(address, layout_entry.nullable(), cs.length, cs.is_varying)
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1421 : : );
1422 : : /* Omit addresses for character sequences. */
1423 : 10 : }
1424 : : }
1425 : 10 : },
1426 : 48 : [&](const Date&) { CALL(_I32<L>); },
1427 : 48 : [&](const DateTime&) { CALL(_I64<L>); },
1428 : 0 : [](auto&&) { M_unreachable("invalid type"); },
1429 : 48 : }, *tuple_it->type);
1430 : : #undef CALL
1431 : 48 : }
1432 : : }
1433 : : }
1434 : :
1435 : : /*----- Recursive lambda to emit stride jumps by processing path from leaves (excluding) to the root. -----*/
1436 : 24 : auto emit_stride_jumps = [&](decltype(levels.crbegin()) curr, const decltype(levels.crend()) end) -> void {
1437 : 28 : auto rec = [&](decltype(levels.crbegin()) curr, const decltype(levels.crend()) end, auto rec) -> void {
1438 [ # # # # : 16 : if (curr == end) return;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1439 : :
1440 : 4 : const auto inner = std::prev(curr); // the child INode of `curr`
1441 : 4 : M_insist(curr->num_tuples % inner->num_tuples == 0, "curr must be whole multiple of inner");
1442 : :
1443 : : /*----- Compute remaining stride for this level. -----*/
1444 : 4 : const auto num_repetition_inner = curr->num_tuples / inner->num_tuples;
1445 : 8 : const auto stride_remaining_in_bits = curr->stride_in_bits -
1446 : 4 : num_repetition_inner * inner->stride_in_bits;
1447 : 4 : M_insist(stride_remaining_in_bits % 8 == 0,
1448 : : "remaining stride of INodes must be whole multiple of a byte");
1449 : :
1450 : : /*----- If there is a remaining stride for this level, emit conditional stride jump. -----*/
1451 [ # # # # : 4 : if (const int32_t remaining_stride_in_bytes = stride_remaining_in_bits / 8) [[likely]] {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1452 : 4 : M_insist(curr->num_tuples > 0);
1453 [ # # # # : 4 : if (curr->num_tuples != 1U) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1454 [ # # # # : 4 : Boolx1 cond_mod = (tuple_id % uint32_t(curr->num_tuples)).eqz();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1455 [ # # # # : 4 : Boolx1 cond_and = (tuple_id bitand uint32_t(curr->num_tuples - 1U)).eqz();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1456 [ # # # # : 4 : const bool use_and = is_pow_2(curr->num_tuples) and options::remainder_removal;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1457 [ # # # # : 4 : Boolx1 cond = use_and ? cond_and : cond_mod; // select implementation to use...
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1458 [ # # # # : 4 : (use_and ? cond_mod : cond_and).discard(); // ... and discard the other
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1459 : :
1460 : : /*----- Emit conditional stride jumps. -----*/
1461 [ # # # # : 8 : IF (cond) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1462 [ # # # # : 16 : for (auto &[_, value] : loading_context) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1463 [ # # # # : 12 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1464 : 0 : M_insist(bool(pred));
1465 [ # # # # : 0 : value.ptr += Select(*pred, remaining_stride_in_bytes, 0); // possibly emit stride jump
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1466 : 0 : } else {
1467 : 24 : value.ptr += remaining_stride_in_bytes; // emit stride jump
1468 : : }
1469 : : }
1470 [ # # # # : 4 : if (null_bitmap_ptr) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1471 [ # # # # : 2 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1472 : 0 : M_insist(bool(pred));
1473 [ # # # # : 0 : *null_bitmap_ptr +=
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1474 : 0 : Select(*pred, remaining_stride_in_bytes, 0); // possibly emit stride jump
1475 : 0 : } else {
1476 : 2 : *null_bitmap_ptr += remaining_stride_in_bytes; // emit stride jump
1477 : : }
1478 : 2 : }
1479 : :
1480 : : /*----- Recurse within IF. -----*/
1481 : 4 : rec(std::next(curr), end, rec);
1482 : 4 : };
1483 : 4 : } else {
1484 [ # # # # : 0 : for (auto &[_, value] : loading_context) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1485 [ # # # # : 0 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1486 : 0 : M_insist(bool(pred));
1487 [ # # # # : 0 : value.ptr += Select(*pred, remaining_stride_in_bytes, 0); // possibly emit stride jump
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1488 : 0 : } else {
1489 : 0 : value.ptr += remaining_stride_in_bytes; // emit stride jump
1490 : : }
1491 : : }
1492 [ # # # # : 0 : if (null_bitmap_ptr) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1493 [ # # # # : 0 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1494 : 0 : M_insist(bool(pred));
1495 [ # # # # : 0 : *null_bitmap_ptr +=
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1496 : 0 : Select(*pred, remaining_stride_in_bytes, 0); // possibly emit stride jump
1497 : 0 : } else {
1498 : 0 : *null_bitmap_ptr += remaining_stride_in_bytes; // emit stride jump
1499 : : }
1500 : 0 : }
1501 : :
1502 : : /*----- Recurse within IF. -----*/
1503 : 0 : rec(std::next(curr), end, rec);
1504 : : }
1505 : 4 : } else {
1506 : : /*----- Recurse without IF. -----*/
1507 : 0 : rec(std::next(curr), end, rec);
1508 : : }
1509 : :
1510 : 16 : };
1511 : 12 : rec(curr, end, rec);
1512 : 12 : };
1513 : :
1514 : : /*----- Process path from DataLayout leaves to the root to emit stride jumps. -----*/
1515 [ # # # # : 24 : BLOCK_OPEN(jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1516 : : /*----- Emit the per-leaf stride jumps, i.e. from one instance of the leaf to the next. -----*/
1517 [ # # # # : 130 : for (auto &[key, value] : loading_context) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1518 : 44 : const uint8_t bit_stride = key.second % 8;
1519 : 44 : const int32_t byte_stride = key.second / 8;
1520 [ # # # # : 44 : if (bit_stride) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1521 [ # # # # : 10 : M_insist(L == 1);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1522 [ # # # # : 10 : M_insist(bool(value.mask));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1523 [ # # # # : 10 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1524 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1525 [ # # # # : 0 : *value.mask <<= Select(*pred, bit_stride, uint8_t(0)); // possibly advance mask
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1526 : 0 : } else {
1527 [ # # # # : 10 : *value.mask <<= bit_stride; // advance mask
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1528 : : }
1529 : : /* If the mask surpasses the first byte, advance pointer to the next byte... */
1530 [ # # # # : 10 : value.ptr += (*value.mask bitand 0xffU).eqz().template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1531 : : /* ... and remove the lowest byte from the mask. */
1532 [ # # # # : 10 : *value.mask = Select((*value.mask bitand 0xffU).eqz(), *value.mask >> 8U, *value.mask);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1533 : 10 : }
1534 [ # # # # : 44 : if (byte_stride) [[likely]] {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1535 [ # # # # : 24 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1536 [ # # # # : 0 : M_insist(L == 1);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1537 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1538 [ # # # # : 0 : value.ptr += Select(*pred, byte_stride, 0); // possibly advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1539 : 0 : } else {
1540 [ # # # # : 48 : value.ptr += int32_t(L) * byte_stride; // advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1541 : : }
1542 : 24 : }
1543 : : }
1544 : : /* Omit the leaf stride jump for the NULL bitmap as it is already done together with the loading. */
1545 : :
1546 [ # # # # : 12 : if (not levels.empty()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1547 : : /*----- Emit the stride jumps between each leaf to the beginning of the parent INode. -----*/
1548 [ # # # # : 12 : Block lowest_inode_jumps(false);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1549 [ # # # # : 180 : for (auto &[key, value] : loading_context) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1550 [ # # # # : 44 : M_insist(levels.back().stride_in_bits % 8 == 0,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1551 : : "stride of INodes must be multiples of a whole byte");
1552 : 88 : const auto stride_remaining_in_bits = levels.back().stride_in_bits -
1553 : 88 : levels.back().num_tuples * key.second;
1554 : 44 : const uint8_t remaining_bit_stride = stride_remaining_in_bits % 8;
1555 : 44 : const int32_t remaining_byte_stride = stride_remaining_in_bits / 8;
1556 [ # # # # : 44 : if (remaining_bit_stride) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1557 [ # # # # : 10 : M_insist(L == 1);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1558 [ # # # # : 10 : M_insist(bool(value.mask));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1559 [ # # # # : 20 : BLOCK_OPEN(lowest_inode_jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1560 : 20 : const uint8_t end_bit_offset = (key.first + levels.back().num_tuples * key.second) % 8;
1561 [ # # # # : 20 : M_insist(end_bit_offset != key.first);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1562 : : /* Reset the mask to initial bit offset... */
1563 [ # # # # : 10 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1564 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1565 [ # # # # : 0 : Wasm_insist(*pred or *value.mask == 1U << key.first,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1566 : : "if the predicate is not fulfilled, the mask should not be advanced");
1567 : 0 : }
1568 [ # # # # : 20 : *value.mask = 1U << key.first;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1569 : : /* ... and advance pointer to next byte if resetting of the mask surpasses the current byte. */
1570 [ # # # # : 10 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1571 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1572 [ # # # # : 0 : value.ptr += Select(*pred, int32_t(end_bit_offset > key.first), 0);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1573 : 0 : } else {
1574 [ # # # # : 30 : value.ptr += int32_t(end_bit_offset > key.first);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1575 : : }
1576 : : }
1577 : 10 : }
1578 [ # # # # : 44 : if (remaining_byte_stride) [[likely]] {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1579 [ # # # # : 88 : BLOCK_OPEN(lowest_inode_jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1580 [ # # # # : 44 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1581 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1582 [ # # # # : 0 : value.ptr +=
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1583 [ # # # # : 0 : Select(*pred, remaining_byte_stride, 0); // possibly advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1584 : 0 : } else {
1585 [ # # # # : 44 : value.ptr += remaining_byte_stride; // advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1586 : : }
1587 : : }
1588 : 44 : }
1589 : : }
1590 [ # # # # : 12 : if (null_bitmap_ptr) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1591 [ # # # # : 4 : M_insist(L == 1);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1592 [ # # # # : 4 : M_insist(bool(null_bitmap_mask));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1593 [ # # # # : 4 : M_insist(levels.back().stride_in_bits % 8 == 0,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1594 : : "stride of INodes must be multiples of a whole byte");
1595 : 8 : const auto stride_remaining_in_bits = levels.back().stride_in_bits -
1596 : 4 : levels.back().num_tuples * null_bitmap_stride_in_bits;
1597 : 4 : const uint8_t remaining_bit_stride = stride_remaining_in_bits % 8;
1598 : 4 : const int32_t remaining_byte_stride = stride_remaining_in_bits / 8;
1599 [ # # # # : 4 : if (remaining_bit_stride) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1600 [ # # # # : 8 : BLOCK_OPEN(lowest_inode_jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1601 : 4 : const uint8_t end_bit_offset =
1602 : 4 : (null_bitmap_bit_offset + levels.back().num_tuples * null_bitmap_stride_in_bits) % 8;
1603 [ # # # # : 4 : M_insist(end_bit_offset != null_bitmap_bit_offset);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1604 : : /* Reset the mask to initial bit offset... */
1605 [ # # # # : 4 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1606 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1607 [ # # # # : 0 : Wasm_insist(*pred or *null_bitmap_mask == 1U << null_bitmap_bit_offset,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1608 : : "if the predicate is not fulfilled, the mask should not be advanced");
1609 : 0 : }
1610 [ # # # # : 4 : *null_bitmap_mask = 1U << null_bitmap_bit_offset;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1611 : : /* ... and advance pointer to next byte if resetting of the mask surpasses the current byte. */
1612 [ # # # # : 4 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1613 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1614 [ # # # # : 0 : *null_bitmap_ptr +=
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1615 [ # # # # : 0 : Select(*pred, int32_t(end_bit_offset > null_bitmap_bit_offset), 0);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1616 : 0 : } else {
1617 [ # # # # : 4 : *null_bitmap_ptr += int32_t(end_bit_offset > null_bitmap_bit_offset);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1618 : : }
1619 : : }
1620 : 4 : }
1621 [ # # # # : 4 : if (remaining_byte_stride) [[likely]] {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1622 [ # # # # : 8 : BLOCK_OPEN(lowest_inode_jumps) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1623 [ # # # # : 4 : if (is_predicated) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1624 [ # # # # : 0 : M_insist(bool(pred));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1625 [ # # # # : 0 : *null_bitmap_ptr +=
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1626 [ # # # # : 0 : Select(*pred, remaining_byte_stride, 0); // possibly advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1627 : 0 : } else {
1628 [ # # # # : 4 : *null_bitmap_ptr += remaining_byte_stride; // advance pointer
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1629 : : }
1630 : : }
1631 : 4 : }
1632 : 4 : }
1633 : :
1634 : : /*----- Emit the stride jumps between all INodes starting at the parent of leaves to the root. -----*/
1635 [ # # # # : 12 : if (not lowest_inode_jumps.empty()) [[likely]] {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1636 [ # # # # : 12 : M_insist(levels.back().num_tuples > 0);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1637 [ # # # # : 12 : if (levels.back().num_tuples != 1U) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1638 [ # # # # : 8 : Boolx1 cond_mod = (tuple_id % uint32_t(levels.back().num_tuples)).eqz();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1639 [ # # # # : 8 : Boolx1 cond_and = (tuple_id bitand uint32_t(levels.back().num_tuples - 1U)).eqz();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1640 [ # # # # : 8 : const bool use_and = is_pow_2(levels.back().num_tuples) and options::remainder_removal;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # + + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1641 [ # # # # : 8 : Boolx1 cond = use_and ? cond_and : cond_mod; // select implementation to use...
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1642 [ # # # # : 8 : (use_and ? cond_mod : cond_and).discard(); // ... and discard the other
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
+ + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1643 : :
1644 : : /*----- Emit conditional stride jumps from outermost Block. -----*/
1645 [ # # # # : 16 : IF (cond) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1646 : 8 : lowest_inode_jumps.attach_to_current();
1647 : :
1648 : : /*----- Recurse within IF. -----*/
1649 : 8 : emit_stride_jumps(std::next(levels.crbegin()), levels.crend());
1650 : 8 : };
1651 : 8 : } else {
1652 [ # # # # : 4 : lowest_inode_jumps.attach_to_current();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1653 : :
1654 : : /*----- Recurse within IF. -----*/
1655 [ # # # # : 4 : emit_stride_jumps(std::next(levels.crbegin()), levels.crend());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1656 : : }
1657 : 12 : } else {
1658 : : /*----- Recurse without outermost IF block. -----*/
1659 [ # # # # : 0 : emit_stride_jumps(std::next(levels.crbegin()), levels.crend());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1660 : : }
1661 : 12 : }
1662 : : }
1663 : 12 : });
1664 : :
1665 : : if constexpr (not IsStore) {
1666 : : /*----- Combine actual values and possible NULL bits to a new `SQL_t` and add this to the environment. -----*/
1667 [ + + # # : 70 : for (std::size_t idx = 0; idx != tuple_value_schema.num_entries(); ++idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1668 [ + - # # : 60 : auto &tuple_entry = tuple_value_schema[idx];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1669 [ + - + - : 120 : std::visit(overloaded{
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1670 : 110 : [&]<typename T>(Expr<T, L> value) {
1671 : 100 : BLOCK_OPEN(loads) {
1672 [ - + + - : 100 : if (has_null_bitmap and layout_schema[tuple_entry.id].second.nullable()) {
+ - + - -
+ + - + -
+ - # # #
# # # # #
# # # # #
# # # - +
+ - + - +
- - + + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1673 [ + - + - : 50 : Expr<T, L> combined(value.insist_not_null(), null_bits[idx]);
+ - + - +
- + - # #
# # # # #
# # # # #
+ - + - +
- + - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1674 [ + - + - : 50 : env.add(tuple_entry.id, combined);
- + + - +
- - + # #
# # # # #
# # # # #
+ - + - -
+ + - + -
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1675 : 50 : } else {
1676 [ # # # # : 0 : env.add(tuple_entry.id, value);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1677 : : }
1678 : : }
1679 : 50 : },
1680 : 70 : [&](NChar value) {
1681 : : if constexpr (L == 1) {
1682 : 20 : BLOCK_OPEN(loads) {
1683 [ - + + - : 20 : if (has_null_bitmap and layout_schema[tuple_entry.id].second.nullable()) {
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1684 : : /* introduce variable s.t. uses only load from it */
1685 [ + - + - : 10 : Var<Ptr<Charx1>> combined(Select(null_bits[idx], Ptr<Charx1>::Nullptr(), value.val()));
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1686 [ + - + - : 10 : env.add(tuple_entry.id, NChar(combined, /* can_be_null=*/ true, value.length(),
+ - + - -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1687 : 10 : value.guarantees_terminating_nul()));
1688 : 10 : } else {
1689 [ # # # # : 0 : Var<Ptr<Charx1>> _value(value.val()); // introduce variable s.t. uses only load from it
# # # # #
# # # # #
# # # # #
# # # #
# ]
1690 [ # # # # : 0 : env.add(tuple_entry.id, NChar(_value, /* can_be_null=*/ false, value.length(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1691 : 0 : value.guarantees_terminating_nul()));
1692 : 0 : }
1693 : : }
1694 : : } else {
1695 : 0 : M_unreachable("string SIMDfication currently not supported");
1696 : : }
1697 : 10 : },
1698 : 0 : [](auto) { M_unreachable("value must be loaded beforehand"); },
1699 : 0 : [](std::monostate) { M_unreachable("invalid variant"); },
1700 : 60 : }, values[idx]);
1701 : 60 : }
1702 : :
1703 : : /*----- Add addresses to the environment. -----*/
1704 [ - + # # : 10 : for (std::size_t idx = 0; idx != tuple_addr_schema.num_entries(); ++idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1705 [ # # # # : 0 : BLOCK_OPEN(loads) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1706 [ # # # # : 0 : auto &tuple_entry = tuple_addr_schema[idx];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1707 [ # # # # : 0 : env.add_addr(tuple_entry.id, std::move(addrs[idx]));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1708 : : }
1709 : 0 : }
1710 : : }
1711 : :
1712 : : /*----- Destroy created values. -----*/
1713 [ # # # # : 70 : for (std::size_t idx = 0; idx < tuple_value_schema.num_entries(); ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1714 : 60 : values[idx].~SQL_t();
1715 [ # # # # : 10 : for (std::size_t idx = 0; idx < tuple_addr_schema.num_entries(); ++idx)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1716 : 0 : addrs[idx].~SQL_addr_t();
1717 : : if constexpr (not IsStore) {
1718 : : /*----- Destroy created NULL bits. -----*/
1719 [ + + # # : 70 : for (std::size_t idx = 0; idx != tuple_value_schema.num_entries(); ++idx) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1720 [ - + + - : 120 : if (has_null_bitmap and layout_schema[tuple_value_schema[idx].id].second.nullable())
+ - + - -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1721 : 60 : null_bits[idx].~Bool<L>();
1722 : 60 : }
1723 : : }
1724 [ # # # # : 10 : base_address.discard(); // discard base address (as it was always cloned)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1725 : :
1726 : : #ifndef NDEBUG
1727 : : if constexpr (IsStore)
1728 [ # # # # : 0 : M_insist(loads.empty());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1729 : : else
1730 [ + - + - : 10 : M_insist(stores.empty());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1731 : : #endif
1732 : :
1733 : : if constexpr (IsStore)
1734 [ # # # # : 0 : return std::make_tuple<Block, Block, Block>(std::move(inits), std::move(stores), std::move(jumps));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1735 : : else
1736 [ + - # # : 10 : return std::make_tuple<Block, Block, Block>(std::move(inits), std::move(loads), std::move(jumps));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1737 : 10 : }
1738 : :
1739 : : }
1740 : :
1741 : : }
1742 : :
1743 : : template<VariableKind Kind>
1744 : : std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block>
1745 : 0 : m::wasm::compile_store_sequential(const Schema &tuple_value_schema, const Schema &tuple_addr_schema,
1746 : : Ptr<void> base_address, const storage::DataLayout &layout, std::size_t num_simd_lanes,
1747 : : const Schema &layout_schema, Variable<uint32_t, Kind, false> &tuple_id)
1748 : : {
1749 [ # # # # : 0 : if (options::pointer_sharing) {
# # ]
1750 [ # # # # : 0 : switch (num_simd_lanes) {
# # # # #
# # # # #
# # # # #
# # ]
1751 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
1752 [ # # # # : 0 : case 1: return compile_data_layout_sequential<true, 1, false, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1753 : 0 : base_address, layout, layout_schema,
1754 : 0 : tuple_id);
1755 [ # # # # : 0 : case 2: return compile_data_layout_sequential<true, 2, false, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1756 : 0 : base_address, layout, layout_schema,
1757 : 0 : tuple_id);
1758 [ # # # # : 0 : case 4: return compile_data_layout_sequential<true, 4, false, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1759 : 0 : base_address, layout, layout_schema,
1760 : 0 : tuple_id);
1761 [ # # # # : 0 : case 8: return compile_data_layout_sequential<true, 8, false, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1762 : 0 : base_address, layout, layout_schema,
1763 : 0 : tuple_id);
1764 [ # # # # : 0 : case 16: return compile_data_layout_sequential<true, 16, false, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1765 : 0 : base_address, layout, layout_schema,
1766 : 0 : tuple_id);
1767 [ # # # # : 0 : case 32: return compile_data_layout_sequential<true, 32, false, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1768 : 0 : base_address, layout, layout_schema,
1769 : 0 : tuple_id);
1770 : : }
1771 : : } else {
1772 [ # # # # : 0 : switch (num_simd_lanes) {
# # # # #
# # # # #
# # # # #
# # ]
1773 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
1774 [ # # # # : 0 : case 1: return compile_data_layout_sequential<true, 1, false, false>(tuple_value_schema,
# # # # #
# # # ]
1775 : 0 : tuple_addr_schema, base_address,
1776 : 0 : layout, layout_schema, tuple_id);
1777 [ # # # # : 0 : case 2: return compile_data_layout_sequential<true, 2, false, false>(tuple_value_schema,
# # # # #
# # # ]
1778 : 0 : tuple_addr_schema, base_address,
1779 : 0 : layout, layout_schema, tuple_id);
1780 [ # # # # : 0 : case 4: return compile_data_layout_sequential<true, 4, false, false>(tuple_value_schema,
# # # # #
# # # ]
1781 : 0 : tuple_addr_schema, base_address,
1782 : 0 : layout, layout_schema, tuple_id);
1783 [ # # # # : 0 : case 8: return compile_data_layout_sequential<true, 8, false, false>(tuple_value_schema,
# # # # #
# # # ]
1784 : 0 : tuple_addr_schema, base_address,
1785 : 0 : layout, layout_schema, tuple_id);
1786 [ # # # # : 0 : case 16: return compile_data_layout_sequential<true, 16, false, false>(tuple_value_schema,
# # # # #
# # # ]
1787 : 0 : tuple_addr_schema, base_address,
1788 : 0 : layout, layout_schema, tuple_id);
1789 [ # # # # : 0 : case 32: return compile_data_layout_sequential<true, 32, false, false>(tuple_value_schema,
# # # # #
# # # ]
1790 : 0 : tuple_addr_schema, base_address,
1791 : 0 : layout, layout_schema, tuple_id);
1792 : : }
1793 : : }
1794 : 0 : }
1795 : :
1796 : : template<VariableKind Kind>
1797 : : std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block>
1798 : 0 : m::wasm::compile_store_sequential_single_pass(const Schema &tuple_value_schema, const Schema &tuple_addr_schema,
1799 : : Ptr<void> base_address, const storage::DataLayout &layout,
1800 : : std::size_t num_simd_lanes, const Schema &layout_schema,
1801 : : Variable<uint32_t, Kind, false> &tuple_id)
1802 : : {
1803 [ # # # # : 0 : if (options::pointer_sharing) {
# # ]
1804 [ # # # # : 0 : switch (num_simd_lanes) {
# # # # #
# # # # #
# # # # #
# # ]
1805 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
1806 [ # # # # : 0 : case 1: return compile_data_layout_sequential<true, 1, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1807 : 0 : base_address, layout, layout_schema,
1808 : 0 : tuple_id);
1809 [ # # # # : 0 : case 2: return compile_data_layout_sequential<true, 2, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1810 : 0 : base_address, layout, layout_schema,
1811 : 0 : tuple_id);
1812 [ # # # # : 0 : case 4: return compile_data_layout_sequential<true, 4, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1813 : 0 : base_address, layout, layout_schema,
1814 : 0 : tuple_id);
1815 [ # # # # : 0 : case 8: return compile_data_layout_sequential<true, 8, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1816 : 0 : base_address, layout, layout_schema,
1817 : 0 : tuple_id);
1818 [ # # # # : 0 : case 16: return compile_data_layout_sequential<true, 16, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1819 : 0 : base_address, layout, layout_schema,
1820 : 0 : tuple_id);
1821 [ # # # # : 0 : case 32: return compile_data_layout_sequential<true, 32, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1822 : 0 : base_address, layout, layout_schema,
1823 : 0 : tuple_id);
1824 : : }
1825 : : } else {
1826 [ # # # # : 0 : switch (num_simd_lanes) {
# # # # #
# # # # #
# # # # #
# # ]
1827 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
1828 [ # # # # : 0 : case 1: return compile_data_layout_sequential<true, 1, true, false>(tuple_value_schema,
# # # # #
# # # ]
1829 : 0 : tuple_addr_schema, base_address,
1830 : 0 : layout, layout_schema, tuple_id);
1831 [ # # # # : 0 : case 2: return compile_data_layout_sequential<true, 2, true, false>(tuple_value_schema,
# # # # #
# # # ]
1832 : 0 : tuple_addr_schema, base_address,
1833 : 0 : layout, layout_schema, tuple_id);
1834 [ # # # # : 0 : case 4: return compile_data_layout_sequential<true, 4, true, false>(tuple_value_schema,
# # # # #
# # # ]
1835 : 0 : tuple_addr_schema, base_address,
1836 : 0 : layout, layout_schema, tuple_id);
1837 [ # # # # : 0 : case 8: return compile_data_layout_sequential<true, 8, true, false>(tuple_value_schema,
# # # # #
# # # ]
1838 : 0 : tuple_addr_schema, base_address,
1839 : 0 : layout, layout_schema, tuple_id);
1840 [ # # # # : 0 : case 16: return compile_data_layout_sequential<true, 16, true, false>(tuple_value_schema,
# # # # #
# # # ]
1841 : 0 : tuple_addr_schema, base_address,
1842 : 0 : layout, layout_schema, tuple_id);
1843 [ # # # # : 0 : case 32: return compile_data_layout_sequential<true, 32, true, false>(tuple_value_schema,
# # # # #
# # # ]
1844 : 0 : tuple_addr_schema, base_address,
1845 : 0 : layout, layout_schema, tuple_id);
1846 : : }
1847 : : }
1848 : 0 : }
1849 : :
1850 : : template<VariableKind Kind>
1851 : : std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block>
1852 : 10 : m::wasm::compile_load_sequential(const Schema &tuple_value_schema, const Schema &tuple_addr_schema,
1853 : : Ptr<void> base_address, const storage::DataLayout &layout, std::size_t num_simd_lanes,
1854 : : const Schema &layout_schema, Variable<uint32_t, Kind, false> &tuple_id)
1855 : : {
1856 [ + - # # : 10 : if (options::pointer_sharing) {
# # ]
1857 [ - + - - : 10 : switch (num_simd_lanes) {
- - - # #
# # # # #
# # # # #
# # ]
1858 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
1859 [ + - + - : 20 : case 1: return compile_data_layout_sequential<false, 1, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1860 : 10 : base_address, layout, layout_schema,
1861 : 10 : tuple_id);
1862 [ # # # # : 0 : case 2: return compile_data_layout_sequential<false, 2, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1863 : 0 : base_address, layout, layout_schema,
1864 : 0 : tuple_id);
1865 [ # # # # : 0 : case 4: return compile_data_layout_sequential<false, 4, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1866 : 0 : base_address, layout, layout_schema,
1867 : 0 : tuple_id);
1868 [ # # # # : 0 : case 8: return compile_data_layout_sequential<false, 8, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1869 : 0 : base_address, layout, layout_schema,
1870 : 0 : tuple_id);
1871 [ # # # # : 0 : case 16: return compile_data_layout_sequential<false, 16, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1872 : 0 : base_address, layout, layout_schema,
1873 : 0 : tuple_id);
1874 [ # # # # : 0 : case 32: return compile_data_layout_sequential<false, 32, true, true>(tuple_value_schema, tuple_addr_schema,
# # # # #
# # # ]
1875 : 0 : base_address, layout, layout_schema,
1876 : 0 : tuple_id);
1877 : : }
1878 : : } else {
1879 [ # # # # : 0 : switch (num_simd_lanes) {
# # # # #
# # # # #
# # # # #
# # ]
1880 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
1881 [ # # # # : 0 : case 1: return compile_data_layout_sequential<false, 1, true, false>(tuple_value_schema,
# # # # #
# # # ]
1882 : 0 : tuple_addr_schema, base_address,
1883 : 0 : layout, layout_schema, tuple_id);
1884 [ # # # # : 0 : case 2: return compile_data_layout_sequential<false, 2, true, false>(tuple_value_schema,
# # # # #
# # # ]
1885 : 0 : tuple_addr_schema, base_address,
1886 : 0 : layout, layout_schema, tuple_id);
1887 [ # # # # : 0 : case 4: return compile_data_layout_sequential<false, 4, true, false>(tuple_value_schema,
# # # # #
# # # ]
1888 : 0 : tuple_addr_schema, base_address,
1889 : 0 : layout, layout_schema, tuple_id);
1890 [ # # # # : 0 : case 8: return compile_data_layout_sequential<false, 8, true, false>(tuple_value_schema,
# # # # #
# # # ]
1891 : 0 : tuple_addr_schema, base_address,
1892 : 0 : layout, layout_schema, tuple_id);
1893 [ # # # # : 0 : case 16: return compile_data_layout_sequential<false, 16, true, false>(tuple_value_schema,
# # # # #
# # # ]
1894 : 0 : tuple_addr_schema, base_address,
1895 : 0 : layout, layout_schema, tuple_id);
1896 [ # # # # : 0 : case 32: return compile_data_layout_sequential<false, 32, true, false>(tuple_value_schema,
# # # # #
# # # ]
1897 : 0 : tuple_addr_schema, base_address,
1898 : 0 : layout, layout_schema, tuple_id);
1899 : : }
1900 : : }
1901 : 10 : }
1902 : :
1903 : : // explicit instantiations to prevent linker errors
1904 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_store_sequential(
1905 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&, Var<U32x1>&
1906 : : );
1907 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_store_sequential(
1908 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&, Global<U32x1>&
1909 : : );
1910 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_store_sequential(
1911 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&,
1912 : : Variable<uint32_t, VariableKind::Param, false>&
1913 : : );
1914 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_store_sequential_single_pass(
1915 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&, Var<U32x1>&
1916 : : );
1917 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_store_sequential_single_pass(
1918 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&, Global<U32x1>&
1919 : : );
1920 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_store_sequential_single_pass(
1921 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&,
1922 : : Variable<uint32_t, VariableKind::Param, false>&
1923 : : );
1924 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_load_sequential(
1925 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&, Var<U32x1>&
1926 : : );
1927 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_load_sequential(
1928 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&, Global<U32x1>&
1929 : : );
1930 : : template std::tuple<m::wasm::Block, m::wasm::Block, m::wasm::Block> m::wasm::compile_load_sequential(
1931 : : const Schema&, const Schema&, Ptr<void>, const storage::DataLayout&, std::size_t, const Schema&,
1932 : : Variable<uint32_t, VariableKind::Param, false>&
1933 : : );
1934 : :
1935 : : namespace m {
1936 : :
1937 : : namespace wasm {
1938 : :
1939 : : /** Compiles the data layout \p layout starting at memory address \p base_address and containing tuples of schema
1940 : : * \p layout_schema such that it stores/loads the single tuple with schema \p _tuple_value_schema and ID \p tuple_id.
1941 : : *
1942 : : * If \tparam IsStore, emits the storing code into the current block, otherwise, emits the loading code into the current
1943 : : * block and adds the loaded values into the current environment. Additionally, if not \tparam IsStore, adds the
1944 : : * addresses of the values of tuples with schema \p _tuple_addr_schema into the current environment. */
1945 : : template<bool IsStore>
1946 : 0 : void compile_data_layout_point_access(const Schema &_tuple_value_schema, const Schema &_tuple_addr_schema,
1947 : : Ptr<void> base_address, const storage::DataLayout &layout,
1948 : : const Schema &layout_schema, U32x1 tuple_id)
1949 : : {
1950 [ # # # # ]: 0 : const auto tuple_value_schema = _tuple_value_schema.deduplicate().drop_constants();
1951 [ # # # # : 0 : const auto tuple_addr_schema = _tuple_addr_schema.deduplicate().drop_constants();
# # # # ]
1952 : :
1953 [ # # # # ]: 0 : M_insist(tuple_value_schema.num_entries() != 0, "point access must access at least one tuple schema entry");
1954 [ # # # # ]: 0 : M_insist(not IsStore or tuple_addr_schema.num_entries() == 0, "addresses are only computed for loads");
1955 : : #ifndef NDEBUG
1956 [ # # # # ]: 0 : for (auto &e : tuple_value_schema)
1957 [ # # # # : 0 : M_insist(layout_schema.find(e.id) != layout_schema.cend(), "tuple value schema entry not found");
# # # # ]
1958 [ # # # # ]: 0 : for (auto &e : tuple_addr_schema) {
1959 [ # # # # ]: 0 : auto it = layout_schema.find(e.id);
1960 [ # # # # ]: 0 : M_insist(it != layout_schema.cend(), "tuple address schema entry not found");
1961 [ # # # # : 0 : M_insist(not it->nullable(), "nullable tuple address schema entry not yet supported");
# # # # ]
1962 [ # # # # : 0 : M_insist(not it->type->is_boolean(), "boolean tuple address schema entry not yet supported");
# # # # ]
1963 [ # # # # : 0 : M_insist(not it->type->is_character_sequence(), "character sequence tuple address schema entry omitted");
# # # # ]
1964 : : }
1965 : : #endif
1966 : :
1967 : : ///> the values loaded for the entries in `tuple_value_schema`
1968 [ # # # # ]: 0 : SQL_t values[tuple_value_schema.num_entries()];
1969 : : ///> the addresses for the entries in `tuple_addr_schema`
1970 : : SQL_addr_t *addrs;
1971 [ # # # # ]: 0 : if (not tuple_addr_schema.empty())
1972 : 0 : addrs = static_cast<SQL_addr_t*>(alloca(sizeof(SQL_addr_t) * tuple_addr_schema.num_entries()));
1973 : : ///> the NULL information loaded for the entries in `tuple_value_schema`
1974 : : Boolx1 *null_bits;
1975 : : if constexpr (not IsStore)
1976 : 0 : null_bits = static_cast<Boolx1*>(alloca(sizeof(Boolx1) * tuple_value_schema.num_entries()));
1977 : :
1978 [ # # # # : 0 : auto &env = CodeGenContext::Get().env(); // the current codegen environment
# # # # ]
1979 : :
1980 : : /*----- Check whether any of the entries in `tuple_value_schema` can be NULL, so that we need the NULL bitmap. -----*/
1981 [ # # # # ]: 0 : const bool needs_null_bitmap = [&]() {
1982 [ # # # # ]: 0 : for (auto &tuple_entry : tuple_value_schema) {
1983 [ # # # # ]: 0 : if (layout_schema[tuple_entry.id].second.nullable())
1984 : 0 : return true; // found an entry in `tuple_value_schema` that can be NULL according to `layout_schema`
1985 : : }
1986 : 0 : return false; // no attribute in `tuple_value_schema` can be NULL according to `layout_schema`
1987 : 0 : }();
1988 : 0 : bool has_null_bitmap = false; // indicates whether the data layout specifies a NULL bitmap
1989 : :
1990 : : /*----- Visit the data layout. -----*/
1991 [ # # # # : 0 : layout.for_sibling_leaves([&](const std::vector<DataLayout::leaf_info_t> &leaves,
# # # # ]
1992 : : const DataLayout::level_info_stack_t &levels, uint64_t inode_offset_in_bits)
1993 : : {
1994 : : /*----- Compute INode pointer and INode iteration depending on the given tuple ID. -----*/
1995 : 0 : auto compute_additional_inode_byte_offset = [&](U32x1 tuple_id) -> U64x1 {
1996 : 0 : auto rec = [&](U32x1 curr_tuple_id, decltype(levels.cbegin()) curr, const decltype(levels.cend()) end,
1997 : : auto rec) -> U64x1
1998 : : {
1999 [ # # # # ]: 0 : if (curr == end) {
2000 [ # # # # : 0 : Wasm_insist(curr_tuple_id == tuple_id % uint32_t(levels.back().num_tuples));
# # # # ]
2001 : 0 : return U64x1(0);
2002 : : }
2003 : :
2004 [ # # # # ]: 0 : if (is_pow_2(curr->num_tuples)) {
2005 [ # # # # ]: 0 : U32x1 child_iter = curr_tuple_id.clone() >> uint32_t(__builtin_ctzl(curr->num_tuples));
2006 [ # # # # ]: 0 : U32x1 inner_tuple_id = curr_tuple_id bitand uint32_t(curr->num_tuples - 1U);
2007 [ # # # # ]: 0 : M_insist(curr->stride_in_bits % 8 == 0, "INode stride must be byte aligned");
2008 [ # # # # ]: 0 : U64x1 offset_in_bytes = child_iter * uint64_t(curr->stride_in_bits / 8);
2009 [ # # # # : 0 : return offset_in_bytes + rec(inner_tuple_id, std::next(curr), end, rec);
# # # # #
# # # # #
# # ]
2010 : 0 : } else {
2011 [ # # # # ]: 0 : U32x1 child_iter = curr_tuple_id.clone() / uint32_t(curr->num_tuples);
2012 [ # # # # ]: 0 : U32x1 inner_tuple_id = curr_tuple_id % uint32_t(curr->num_tuples);
2013 [ # # # # ]: 0 : M_insist(curr->stride_in_bits % 8 == 0, "INode stride must be byte aligned");
2014 [ # # # # ]: 0 : U64x1 offset_in_bytes = child_iter * uint64_t(curr->stride_in_bits / 8);
2015 [ # # # # : 0 : return offset_in_bytes + rec(inner_tuple_id, std::next(curr), end, rec);
# # # # #
# # # # #
# # ]
2016 : 0 : }
2017 : 0 : };
2018 [ # # # # ]: 0 : return rec(tuple_id.clone(), levels.cbegin(), levels.cend(), rec);
2019 : 0 : };
2020 : 0 : M_insist(inode_offset_in_bits % 8 == 0, "INode offset must be byte aligned");
2021 [ # # # # ]: 0 : const Var<Ptr<void>> inode_ptr(
2022 : 0 : base_address.clone()
2023 [ # # # # ]: 0 : + int32_t(inode_offset_in_bits / 8)
2024 [ # # # # : 0 : + compute_additional_inode_byte_offset(tuple_id.clone()).make_signed().template to<int32_t>()
# # # # #
# # # # #
# # # # #
# ]
2025 : : );
2026 : 0 : std::optional<const Var<U32x1>> inode_iter;
2027 [ # # # # ]: 0 : M_insist(levels.back().num_tuples != 0, "INode must be large enough for at least one tuple");
2028 [ # # # # ]: 0 : if (levels.back().num_tuples != 1) {
2029 [ # # # # ]: 0 : inode_iter.emplace(
2030 [ # # # # : 0 : is_pow_2(levels.back().num_tuples) ? tuple_id bitand uint32_t(levels.back().num_tuples - 1U)
# # # # ]
2031 [ # # # # ]: 0 : : tuple_id % uint32_t(levels.back().num_tuples)
2032 : : );
2033 : 0 : } else {
2034 : : /* omit computation of INode iteration since it is always the first iteration, i.e. equals 0 */
2035 [ # # # # ]: 0 : tuple_id.discard();
2036 : : }
2037 : :
2038 : : /*----- Iterate over sibling leaves, i.e. leaf children of a common parent INode, to emit code. -----*/
2039 [ # # # # ]: 0 : for (auto &leaf_info : leaves) {
2040 : 0 : const uint8_t bit_stride = leaf_info.stride_in_bits % 8;
2041 : :
2042 [ # # # # ]: 0 : if (leaf_info.leaf.index() == layout_schema.num_entries()) { // NULL bitmap
2043 [ # # # # ]: 0 : if (not needs_null_bitmap)
2044 : 0 : continue;
2045 : :
2046 [ # # # # ]: 0 : M_insist(not has_null_bitmap, "at most one bitmap may be specified");
2047 : 0 : has_null_bitmap = true;
2048 [ # # # # ]: 0 : if (bit_stride) { // NULL bitmap with bit stride requires dynamic masking
2049 [ # # # # ]: 0 : M_insist(bool(inode_iter), "stride requires repetition");
2050 [ # # # # : 0 : U64x1 leaf_offset_in_bits = leaf_info.offset_in_bits + *inode_iter * leaf_info.stride_in_bits;
# # # # ]
2051 [ # # # # ]: 0 : const Var<U8x1> leaf_bit_offset(
2052 [ # # # # : 0 : (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>() // mod 8
# # # # #
# # # ]
2053 : : );
2054 [ # # # # : 0 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # ]
2055 : :
2056 [ # # # # : 0 : const Var<Ptr<void>> ptr(inode_ptr + leaf_byte_offset); // pointer to NULL bitmap
# # # # ]
2057 : :
2058 : : /*----- For each tuple entry that can be NULL, create a store/load with offset and mask. --*/
2059 [ # # # # ]: 0 : for (std::size_t tuple_idx = 0; tuple_idx != tuple_value_schema.num_entries(); ++tuple_idx) {
2060 [ # # # # ]: 0 : auto &tuple_entry = tuple_value_schema[tuple_idx];
2061 [ # # # # ]: 0 : const auto &[layout_idx, layout_entry] = layout_schema[tuple_entry.id];
2062 [ # # # # : 0 : M_insist(*tuple_entry.type == *layout_entry.type);
# # # # #
# # # ]
2063 [ # # # # : 0 : if (layout_entry.nullable()) { // layout entry may be NULL
# # # # ]
2064 [ # # # # ]: 0 : U64x1 offset_in_bits = leaf_bit_offset + layout_idx;
2065 [ # # # # : 0 : U8x1 bit_offset = (offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>() ; // mod 8
# # # # #
# # # ]
2066 [ # # # # : 0 : I32x1 byte_offset = (offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # ]
2067 : : if constexpr (IsStore) {
2068 : : /*----- Store NULL bit depending on its type. -----*/
2069 : 0 : auto store = [&]<typename T>() {
2070 [ # # # # : 0 : auto [value, is_null] = env.get<T>(tuple_entry.id).split(); // get value
# # # # #
# # # #
# ]
2071 [ # # # # : 0 : value.discard(); // handled at entry leaf
# # # # #
# # # #
# ]
2072 : : Ptr<U8x1> byte_ptr =
2073 [ # # # # : 0 : (ptr + byte_offset).template to<uint8_t*>(); // compute byte address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2074 [ # # # # : 0 : setbit<U8x1>(byte_ptr, is_null, uint8_t(1) << bit_offset); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2075 : 0 : };
2076 [ # # # # : 0 : visit(overloaded{
# # # # #
# ]
2077 : 0 : [&](const Boolean&) { store.template operator()<_Boolx1>(); },
2078 : 0 : [&](const Numeric &n) {
2079 [ # # # ]: 0 : switch (n.kind) {
2080 : : case Numeric::N_Int:
2081 : : case Numeric::N_Decimal:
2082 [ # # # # : 0 : switch (n.size()) {
# ]
2083 : 0 : default: M_unreachable("invalid size");
2084 : 0 : case 8: store.template operator()<_I8x1 >(); break;
2085 : 0 : case 16: store.template operator()<_I16x1>(); break;
2086 : 0 : case 32: store.template operator()<_I32x1>(); break;
2087 : 0 : case 64: store.template operator()<_I64x1>(); break;
2088 : : }
2089 : 0 : break;
2090 : : case Numeric::N_Float:
2091 [ # # ]: 0 : if (n.size() <= 32)
2092 : 0 : store.template operator()<_Floatx1>();
2093 : : else
2094 : 0 : store.template operator()<_Doublex1>();
2095 : 0 : }
2096 : 0 : },
2097 : 0 : [&](const CharacterSequence&) {
2098 : 0 : auto value = env.get<NChar>(tuple_entry.id); // get value
2099 : : Ptr<U8x1> byte_ptr =
2100 [ # # # # ]: 0 : (ptr + byte_offset).template to<uint8_t*>(); // compute byte address
2101 [ # # # # : 0 : setbit<U8x1>(byte_ptr, value.is_null(), uint8_t(1) << bit_offset); // update bit
# # # # ]
2102 : 0 : },
2103 : 0 : [&](const Date&) { store.template operator()<_I32x1>(); },
2104 : 0 : [&](const DateTime&) { store.template operator()<_I64x1>(); },
2105 : 0 : [](auto&&) { M_unreachable("invalid type"); },
2106 : 0 : }, *tuple_entry.type);
2107 : : } else {
2108 : : /*----- Load NULL bit. -----*/
2109 [ # # # # : 0 : U8x1 byte = *(ptr + byte_offset).template to<uint8_t*>(); // load the byte
# # # # ]
2110 [ # # # # : 0 : Var<Boolx1> value((byte bitand (uint8_t(1) << bit_offset)).to<bool>()); // mask bit
# # # # ]
2111 [ # # ]: 0 : new (&null_bits[tuple_idx]) Boolx1(value);
2112 : : /* Address for NULL bits not yet supported. */
2113 : 0 : }
2114 : 0 : } else { // entry must not be NULL
2115 : : #ifndef NDEBUG
2116 : : if constexpr (IsStore) {
2117 : : /*----- Check that value is also not NULL. -----*/
2118 : 0 : auto check = [&]<typename T>() {
2119 [ # # # # : 0 : Wasm_insist(env.get<T>(tuple_entry.id).not_null(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2120 : : "value of non-nullable entry must not be nullable");
2121 : 0 : };
2122 [ # # # # : 0 : visit(overloaded{
# # # # #
# ]
2123 : 0 : [&](const Boolean&) { check.template operator()<_Boolx1>(); },
2124 : 0 : [&](const Numeric &n) {
2125 [ # # # ]: 0 : switch (n.kind) {
2126 : : case Numeric::N_Int:
2127 : : case Numeric::N_Decimal:
2128 [ # # # # : 0 : switch (n.size()) {
# ]
2129 : 0 : default: M_unreachable("invalid size");
2130 : 0 : case 8: check.template operator()<_I8x1 >(); break;
2131 : 0 : case 16: check.template operator()<_I16x1>(); break;
2132 : 0 : case 32: check.template operator()<_I32x1>(); break;
2133 : 0 : case 64: check.template operator()<_I64x1>(); break;
2134 : : }
2135 : 0 : break;
2136 : : case Numeric::N_Float:
2137 [ # # ]: 0 : if (n.size() <= 32)
2138 : 0 : check.template operator()<_Floatx1>();
2139 : : else
2140 : 0 : check.template operator()<_Doublex1>();
2141 : 0 : }
2142 : 0 : },
2143 : 0 : [&](const CharacterSequence&) { check.template operator()<NChar>(); },
2144 : 0 : [&](const Date&) { check.template operator()<_I32x1>(); },
2145 : 0 : [&](const DateTime&) { check.template operator()<_I64x1>(); },
2146 : 0 : [](auto&&) { M_unreachable("invalid type"); },
2147 : 0 : }, *tuple_entry.type);
2148 : : }
2149 : : #endif
2150 : : }
2151 : 0 : }
2152 : 0 : } else { // NULL bitmap without bit stride can benefit from static masking of NULL bits
2153 [ # # # # ]: 0 : auto ptr = [&]() -> Ptr<void> {
2154 [ # # # # : 0 : if (inode_iter and leaf_info.stride_in_bits) {
# # # # ]
2155 : : /* omit `leaf_info.offset_in_bits` here to add it to the static offsets and masks;
2156 : : * this is valid since no bit stride means that the leaf byte offset computation is
2157 : : * independent of the static parts */
2158 : 0 : U64x1 leaf_offset_in_bits = *inode_iter * leaf_info.stride_in_bits;
2159 [ # # # # : 0 : U8x1 leaf_bit_offset = (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>(); // mod 8
# # # # #
# # # ]
2160 [ # # # # : 0 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # ]
2161 [ # # # # : 0 : Wasm_insist(leaf_bit_offset == 0U, "no leaf bit offset without bit stride");
# # # # #
# # # ]
2162 [ # # # # : 0 : const Var<Ptr<void>> ptr(inode_ptr + leaf_byte_offset);
# # # # ]
2163 [ # # # # ]: 0 : return ptr;
2164 : 0 : } else {
2165 : 0 : return inode_ptr;
2166 : : }
2167 : 0 : }(); // pointer to NULL bitmap
2168 : :
2169 : : /*----- For each tuple entry that can be NULL, create a store/load with offset and mask. --*/
2170 [ # # # # ]: 0 : for (std::size_t tuple_idx = 0; tuple_idx != tuple_value_schema.num_entries(); ++tuple_idx) {
2171 [ # # # # ]: 0 : auto &tuple_entry = tuple_value_schema[tuple_idx];
2172 [ # # # # ]: 0 : const auto &[layout_idx, layout_entry] = layout_schema[tuple_entry.id];
2173 [ # # # # : 0 : M_insist(*tuple_entry.type == *layout_entry.type);
# # # # #
# # # ]
2174 [ # # # # : 0 : if (layout_entry.nullable()) { // layout entry may be NULL
# # # # ]
2175 : 0 : const uint8_t static_bit_offset = (leaf_info.offset_in_bits + layout_idx) % 8;
2176 : 0 : const int32_t static_byte_offset = (leaf_info.offset_in_bits + layout_idx) / 8;
2177 : : if constexpr (IsStore) {
2178 : : /*----- Store NULL bit depending on its type. -----*/
2179 : 0 : auto store = [&]<typename T>() {
2180 [ # # # # : 0 : auto [value, is_null] = env.get<T>(tuple_entry.id).split(); // get value
# # # # #
# # # #
# ]
2181 [ # # # # : 0 : value.discard(); // handled at entry leaf
# # # # #
# # # #
# ]
2182 : : Ptr<U8x1> byte_ptr =
2183 [ # # # # : 0 : (ptr.clone() + static_byte_offset).template to<uint8_t*>(); // compute byte address
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2184 [ # # # # : 0 : setbit<U8x1>(byte_ptr, is_null, static_bit_offset); // update bit
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2185 : 0 : };
2186 [ # # # # : 0 : visit(overloaded{
# # # # #
# ]
2187 : 0 : [&](const Boolean&) { store.template operator()<_Boolx1>(); },
2188 : 0 : [&](const Numeric &n) {
2189 [ # # # ]: 0 : switch (n.kind) {
2190 : : case Numeric::N_Int:
2191 : : case Numeric::N_Decimal:
2192 [ # # # # : 0 : switch (n.size()) {
# ]
2193 : 0 : default: M_unreachable("invalid size");
2194 : 0 : case 8: store.template operator()<_I8x1 >(); break;
2195 : 0 : case 16: store.template operator()<_I16x1>(); break;
2196 : 0 : case 32: store.template operator()<_I32x1>(); break;
2197 : 0 : case 64: store.template operator()<_I64x1>(); break;
2198 : : }
2199 : 0 : break;
2200 : : case Numeric::N_Float:
2201 [ # # ]: 0 : if (n.size() <= 32)
2202 : 0 : store.template operator()<_Floatx1>();
2203 : : else
2204 : 0 : store.template operator()<_Doublex1>();
2205 : 0 : }
2206 : 0 : },
2207 : 0 : [&](const CharacterSequence&) {
2208 : 0 : auto value = env.get<NChar>(tuple_entry.id); // get value
2209 : : Ptr<U8x1> byte_ptr =
2210 [ # # # # : 0 : (ptr.clone() + static_byte_offset).template to<uint8_t*>(); // compute byte address
# # ]
2211 [ # # # # : 0 : setbit<U8x1>(byte_ptr, value.is_null(), static_bit_offset); // update bit
# # ]
2212 : 0 : },
2213 : 0 : [&](const Date&) { store.template operator()<_I32x1>(); },
2214 : 0 : [&](const DateTime&) { store.template operator()<_I64x1>(); },
2215 : 0 : [](auto&&) { M_unreachable("invalid type"); },
2216 : 0 : }, *tuple_entry.type);
2217 : : } else {
2218 : : /*----- Load NULL bit. -----*/
2219 [ # # # # : 0 : U8x1 byte = *(ptr.clone() + static_byte_offset).template to<uint8_t*>(); // load the byte
# # # # #
# ]
2220 : 0 : const uint8_t static_mask = 1U << static_bit_offset;
2221 [ # # # # : 0 : Var<Boolx1> value((byte bitand static_mask).to<bool>()); // mask bit
# # ]
2222 [ # # ]: 0 : new (&null_bits[tuple_idx]) Boolx1(value);
2223 : : /* Address for NULL bits not yet supported. */
2224 : 0 : }
2225 : 0 : } else { // entry must not be NULL
2226 : : #ifndef NDEBUG
2227 : : if constexpr (IsStore) {
2228 : : /*----- Check that value is also not NULL. -----*/
2229 : 0 : auto check = [&]<typename T>() {
2230 [ # # # # : 0 : Wasm_insist(env.get<T>(tuple_entry.id).not_null(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2231 : : "value of non-nullable entry must not be nullable");
2232 : 0 : };
2233 [ # # # # : 0 : visit(overloaded{
# # # # #
# ]
2234 : 0 : [&](const Boolean&) { check.template operator()<_Boolx1>(); },
2235 : 0 : [&](const Numeric &n) {
2236 [ # # # ]: 0 : switch (n.kind) {
2237 : : case Numeric::N_Int:
2238 : : case Numeric::N_Decimal:
2239 [ # # # # : 0 : switch (n.size()) {
# ]
2240 : 0 : default: M_unreachable("invalid size");
2241 : 0 : case 8: check.template operator()<_I8x1 >(); break;
2242 : 0 : case 16: check.template operator()<_I16x1>(); break;
2243 : 0 : case 32: check.template operator()<_I32x1>(); break;
2244 : 0 : case 64: check.template operator()<_I64x1>(); break;
2245 : : }
2246 : 0 : break;
2247 : : case Numeric::N_Float:
2248 [ # # ]: 0 : if (n.size() <= 32)
2249 : 0 : check.template operator()<_Floatx1>();
2250 : : else
2251 : 0 : check.template operator()<_Doublex1>();
2252 : 0 : }
2253 : 0 : },
2254 : 0 : [&](const CharacterSequence&) { check.template operator()<NChar>(); },
2255 : 0 : [&](const Date&) { check.template operator()<_I32x1>(); },
2256 : 0 : [&](const DateTime&) { check.template operator()<_I64x1>(); },
2257 : 0 : [](auto&&) { M_unreachable("invalid type"); },
2258 : 0 : }, *tuple_entry.type);
2259 : : }
2260 : : #endif
2261 : : }
2262 : 0 : }
2263 [ # # # # ]: 0 : ptr.discard(); // since it was always cloned
2264 : 0 : }
2265 : 0 : } else { // regular entry
2266 [ # # # # ]: 0 : auto &layout_entry = layout_schema[leaf_info.leaf.index()];
2267 [ # # # # : 0 : M_insist(*layout_entry.type == *leaf_info.leaf.type());
# # # # ]
2268 [ # # # # ]: 0 : auto tuple_value_it = tuple_value_schema.find(layout_entry.id);
2269 [ # # # # ]: 0 : auto tuple_addr_it = tuple_addr_schema.find(layout_entry.id);
2270 [ # # # # : 0 : if (tuple_value_it == tuple_value_schema.end() and tuple_addr_it == tuple_addr_schema.end())
# # # # ]
2271 : 0 : continue; // entry not contained in both tuple schemas
2272 [ # # # # ]: 0 : auto tuple_it = tuple_value_it != tuple_value_schema.end() ? tuple_value_it : tuple_addr_it;
2273 [ # # # # : 0 : M_insist(*tuple_it->type == *layout_entry.type);
# # # # ]
2274 [ # # # # ]: 0 : const auto tuple_value_idx = std::distance(tuple_value_schema.begin(), tuple_value_it);
2275 [ # # # # ]: 0 : const auto tuple_addr_idx = std::distance(tuple_addr_schema.begin(), tuple_addr_it);
2276 : :
2277 [ # # # # ]: 0 : if (bit_stride) { // entry with bit stride requires dynamic masking
2278 [ # # # # : 0 : M_insist(tuple_it->type->is_boolean(), "leaf bit stride currently only for `Boolean` supported");
# # # # ]
2279 : :
2280 [ # # # # ]: 0 : M_insist(bool(inode_iter), "stride requires repetition");
2281 [ # # # # : 0 : U64x1 leaf_offset_in_bits = leaf_info.offset_in_bits + *inode_iter * leaf_info.stride_in_bits;
# # # # ]
2282 [ # # # # : 0 : U8x1 leaf_bit_offset = (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>() ; // mod 8
# # # # #
# # # ]
2283 [ # # # # : 0 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # ]
2284 : :
2285 [ # # # # : 0 : Ptr<U8x1> byte_ptr = (inode_ptr + leaf_byte_offset).template to<uint8_t*>();
# # # # ]
2286 [ # # # # ]: 0 : U8x1 mask = uint8_t(1) << leaf_bit_offset;
2287 : :
2288 : : if constexpr (IsStore) {
2289 : : /*----- Store value. -----*/
2290 [ # # # # ]: 0 : auto [value, is_null] = env.get<_Boolx1>(tuple_it->id).split(); // get value
2291 [ # # ]: 0 : is_null.discard(); // handled at NULL bitmap leaf
2292 [ # # # # : 0 : setbit(byte_ptr, value, mask); // update bit
# # # # ]
2293 : 0 : } else {
2294 : : /*----- Load value. -----*/
2295 : : /* TODO: load byte once, create values with respective mask */
2296 [ # # ]: 0 : if (tuple_value_it != tuple_value_schema.end()) {
2297 [ # # # # : 0 : Var<Boolx1> value((*byte_ptr.clone() bitand mask.clone()).template to<bool>()); // mask bit with dynamic mask
# # # # #
# # # ]
2298 [ # # # # : 0 : new (&values[tuple_value_idx]) SQL_t(_Boolx1(value));
# # ]
2299 : 0 : }
2300 : : /* Address for booleans not yet supported. */
2301 [ # # ]: 0 : byte_ptr.discard();
2302 [ # # ]: 0 : mask.discard();
2303 : : }
2304 : 0 : } else { // entry without bit stride; if masking is required, we can use a static mask
2305 [ # # # # ]: 0 : auto ptr = [&]() -> Ptr<void> {
2306 [ # # # # : 0 : if (inode_iter and leaf_info.stride_in_bits) {
# # # # ]
2307 : : /* omit `leaf_info.offset_in_bits` here to use it as static offset and mask;
2308 : : * this is valid since no bit stride means that the leaf byte offset computation is
2309 : : * independent of the static parts */
2310 : 0 : U64x1 leaf_offset_in_bits = *inode_iter * leaf_info.stride_in_bits;
2311 [ # # # # : 0 : U8x1 leaf_bit_offset = (leaf_offset_in_bits.clone() bitand uint64_t(7)).to<uint8_t>(); // mod 8
# # # # #
# # # ]
2312 [ # # # # : 0 : I32x1 leaf_byte_offset = (leaf_offset_in_bits >> uint64_t(3)).make_signed().to<int32_t>(); // div 8
# # # # #
# # # ]
2313 [ # # # # : 0 : Wasm_insist(leaf_bit_offset == 0U, "no leaf bit offset without bit stride");
# # # # #
# # # ]
2314 [ # # # # ]: 0 : return inode_ptr + leaf_byte_offset;
2315 : 0 : } else {
2316 : 0 : return inode_ptr;
2317 : : }
2318 : 0 : }(); // pointer to entry
2319 : :
2320 : 0 : const uint8_t static_bit_offset = leaf_info.offset_in_bits % 8;
2321 : 0 : const int32_t static_byte_offset = leaf_info.offset_in_bits / 8;
2322 : :
2323 : : /*----- Store value depending on its type. -----*/
2324 : 0 : auto store = [&]<typename T>() {
2325 : : using type = typename T::type;
2326 : 0 : M_insist(static_bit_offset == 0,
2327 : : "leaf offset of `Numeric`, `Date`, or `DateTime` must be byte aligned");
2328 [ # # # # : 0 : auto [value, is_null] = env.get<T>(tuple_it->id).split(); // get value
# # # # #
# # # ]
2329 [ # # # # : 0 : is_null.discard(); // handled at NULL bitmap leaf
# # # # #
# # # ]
2330 [ # # # # : 0 : *(ptr + static_byte_offset).template to<type*>() = value;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2331 : 0 : };
2332 : : /*----- Load value depending on its type. -----*/
2333 : 0 : auto load = [&]<typename T>() {
2334 : : using type = typename T::type;
2335 : 0 : M_insist(static_bit_offset == 0,
2336 : : "leaf offset of `Numeric`, `Date`, or `DateTime` must be byte aligned");
2337 [ # # # # : 0 : if (tuple_value_it != tuple_value_schema.end()) {
# # # # #
# # # ]
2338 [ # # # # : 0 : Var<PrimitiveExpr<type>> value(*(ptr.clone() + static_byte_offset).template to<type*>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2339 [ # # # # : 0 : new (&values[tuple_value_idx]) SQL_t(T(value));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2340 : 0 : }
2341 [ # # # # : 0 : if (tuple_addr_it != tuple_addr_schema.end())
# # # # #
# # # ]
2342 [ # # # # : 0 : new (&addrs[tuple_addr_idx]) SQL_addr_t(
# # # # #
# # # ]
2343 [ # # # # : 0 : (ptr.clone() + static_byte_offset).template to<type*>()
# # # # #
# # # # #
# # # # #
# # # #
# ]
2344 : : );
2345 : 0 : ptr.discard();
2346 : 0 : };
2347 : : /*----- Select call target (store or load) and visit attribute type. -----*/
2348 : : #define CALL(TYPE) if constexpr (IsStore) store.template operator()<TYPE>(); else load.template operator()<TYPE>()
2349 [ # # # # : 0 : visit(overloaded{
# # # # #
# # # # #
# # # # #
# ]
2350 : 0 : [&](const Boolean&) {
2351 [ # # # # ]: 0 : Ptr<U8x1> byte_ptr = (ptr + static_byte_offset).template to<uint8_t*>();
2352 : : if constexpr (IsStore) {
2353 : : /*----- Store value. -----*/
2354 [ # # # # ]: 0 : auto [value, is_null] = env.get<_Boolx1>(tuple_it->id).split(); // get value
2355 [ # # ]: 0 : is_null.discard(); // handled at NULL bitmap leaf
2356 [ # # # # : 0 : setbit<U8x1>(byte_ptr, value, static_bit_offset); // update bit
# # ]
2357 : 0 : } else {
2358 : : /*----- Load value. -----*/
2359 : : /* TODO: load byte once, create values with respective mask */
2360 : 0 : const uint8_t static_mask = 1U << static_bit_offset;
2361 : :
2362 [ # # ]: 0 : if (tuple_value_it != tuple_value_schema.end()) {
2363 [ # # # # : 0 : Var<Boolx1> value((*byte_ptr.clone() bitand static_mask).to<bool>()); // mask bit
# # # # #
# ]
2364 [ # # # # : 0 : new (&values[tuple_value_idx]) SQL_t(_Boolx1(value));
# # ]
2365 : 0 : }
2366 : : /* Address for booleans not yet supported. */
2367 [ # # ]: 0 : byte_ptr.discard();
2368 : : }
2369 : 0 : },
2370 : 0 : [&](const Numeric &n) {
2371 [ # # # # : 0 : switch (n.kind) {
# # ]
2372 : : case Numeric::N_Int:
2373 : : case Numeric::N_Decimal:
2374 [ # # # # : 0 : switch (n.size()) {
# # # # #
# ]
2375 : 0 : default: M_unreachable("invalid size");
2376 : 0 : case 8: CALL(_I8x1 ); break;
2377 : 0 : case 16: CALL(_I16x1); break;
2378 : 0 : case 32: CALL(_I32x1); break;
2379 : 0 : case 64: CALL(_I64x1); break;
2380 : : }
2381 : 0 : break;
2382 : : case Numeric::N_Float:
2383 [ # # # # ]: 0 : if (n.size() <= 32)
2384 : 0 : CALL(_Floatx1);
2385 : : else
2386 : 0 : CALL(_Doublex1);
2387 : 0 : }
2388 : 0 : },
2389 : 0 : [&](const CharacterSequence &cs) {
2390 : 0 : M_insist(static_bit_offset == 0, "leaf offset of `CharacterSequence` must be byte aligned");
2391 [ # # # # ]: 0 : Ptr<Charx1> addr = (ptr + static_byte_offset).template to<char*>();
2392 : : if constexpr (IsStore) {
2393 : : /*----- Store value. -----*/
2394 [ # # ]: 0 : auto value = env.get<NChar>(tuple_it->id); // get value
2395 [ # # # # : 0 : IF (value.clone().not_null()) {
# # # # ]
2396 [ # # # # : 0 : strncpy(addr, value, U32x1(cs.size() / 8)).discard();
# # # # #
# ]
2397 : 0 : };
2398 : 0 : } else {
2399 : : /*----- Load value. -----*/
2400 [ # # ]: 0 : new (&values[tuple_value_idx]) SQL_t(
2401 [ # # # # : 0 : NChar(addr, layout_entry.nullable(), cs.length, cs.is_varying)
# # ]
2402 : : );
2403 : : /* Omit addresses for character sequences. */
2404 : : }
2405 : 0 : },
2406 : 0 : [&](const Date&) { CALL(_I32x1); },
2407 : 0 : [&](const DateTime&) { CALL(_I64x1); },
2408 : 0 : [](auto&&) { M_unreachable("invalid type"); },
2409 : 0 : }, *tuple_it->type);
2410 : : #undef CALL
2411 : 0 : }
2412 : : }
2413 : : }
2414 : 0 : });
2415 : :
2416 : : if constexpr (not IsStore) {
2417 : : /*----- Combine actual values and possible NULL bits to a new `SQL_t` and add this to the environment. -----*/
2418 [ # # ]: 0 : for (std::size_t idx = 0; idx != tuple_value_schema.num_entries(); ++idx) {
2419 [ # # ]: 0 : auto &tuple_entry = tuple_value_schema[idx];
2420 [ # # # # ]: 0 : std::visit(overloaded{
2421 : 0 : [&]<typename T>(Expr<T> value) {
2422 [ # # # # : 0 : if (has_null_bitmap and layout_schema[tuple_entry.id].second.nullable()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2423 [ # # # # : 0 : Expr<T> combined(value.insist_not_null(), null_bits[idx]);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2424 [ # # # # : 0 : env.add(tuple_entry.id, combined);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2425 : 0 : } else {
2426 [ # # # # : 0 : env.add(tuple_entry.id, value);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2427 : : }
2428 : 0 : },
2429 : 0 : [&](NChar value) {
2430 [ # # # # ]: 0 : if (has_null_bitmap and layout_schema[tuple_entry.id].second.nullable()) {
2431 : : /* introduce variable s.t. uses only load from it */
2432 [ # # # # : 0 : Var<Ptr<Charx1>> combined(Select(null_bits[idx], Ptr<Charx1>::Nullptr(), value.val()));
# # ]
2433 [ # # # # : 0 : env.add(tuple_entry.id, NChar(combined, /* can_be_null=*/ true, value.length(),
# # # # #
# ]
2434 : 0 : value.guarantees_terminating_nul()));
2435 : 0 : } else {
2436 [ # # ]: 0 : Var<Ptr<Charx1>> _value(value.val()); // introduce variable s.t. uses only load from it
2437 [ # # # # : 0 : env.add(tuple_entry.id, NChar(_value, /* can_be_null=*/ false, value.length(),
# # # # #
# ]
2438 : 0 : value.guarantees_terminating_nul()));
2439 : 0 : }
2440 : 0 : },
2441 : 0 : [](auto) { M_unreachable("SIMDfication currently not supported"); },
2442 : 0 : [](std::monostate) { M_unreachable("value must be loaded beforehand"); },
2443 : 0 : }, values[idx]);
2444 : 0 : }
2445 : :
2446 : : /*----- Add addresses to the environment. -----*/
2447 [ # # ]: 0 : for (std::size_t idx = 0; idx != tuple_addr_schema.num_entries(); ++idx) {
2448 [ # # ]: 0 : auto &tuple_entry = tuple_addr_schema[idx];
2449 [ # # # # ]: 0 : env.add_addr(tuple_entry.id, std::move(addrs[idx]));
2450 : 0 : }
2451 : : }
2452 : :
2453 : : /*----- Destroy created values and addresses. -----*/
2454 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < tuple_value_schema.num_entries(); ++idx)
2455 : 0 : values[idx].~SQL_t();
2456 [ # # # # ]: 0 : for (std::size_t idx = 0; idx < tuple_addr_schema.num_entries(); ++idx)
2457 : 0 : addrs[idx].~SQL_addr_t();
2458 : : if constexpr (not IsStore) {
2459 : : /*----- Destroy created NULL bits. -----*/
2460 [ # # ]: 0 : for (std::size_t idx = 0; idx != tuple_value_schema.num_entries(); ++idx) {
2461 [ # # # # : 0 : if (has_null_bitmap and layout_schema[tuple_value_schema[idx].id].second.nullable())
# # # # #
# ]
2462 : 0 : null_bits[idx].~Boolx1();
2463 : 0 : }
2464 : : }
2465 [ # # # # ]: 0 : base_address.discard(); // discard base address (as it was always cloned)
2466 : 0 : }
2467 : :
2468 : : }
2469 : :
2470 : : }
2471 : :
2472 : 0 : void m::wasm::compile_store_point_access(const Schema &tuple_value_schema, const Schema &tuple_addr_schema,
2473 : : Ptr<void> base_address, const DataLayout &layout, const Schema &layout_schema,
2474 : : U32x1 tuple_id)
2475 : : {
2476 [ # # ]: 0 : return compile_data_layout_point_access<true>(tuple_value_schema, tuple_addr_schema, base_address, layout,
2477 [ # # ]: 0 : layout_schema, tuple_id);
2478 : 0 : }
2479 : :
2480 : 0 : void m::wasm::compile_load_point_access(const Schema &tuple_value_schema, const Schema &tuple_addr_schema,
2481 : : Ptr<void> base_address, const DataLayout &layout, const Schema &layout_schema,
2482 : : U32x1 tuple_id)
2483 : : {
2484 [ # # ]: 0 : return compile_data_layout_point_access<false>(tuple_value_schema, tuple_addr_schema, base_address, layout,
2485 [ # # ]: 0 : layout_schema, tuple_id);
2486 : 0 : }
2487 : :
2488 : :
2489 : : /*======================================================================================================================
2490 : : * Buffer
2491 : : *====================================================================================================================*/
2492 : :
2493 : : template<bool IsGlobal>
2494 : 0 : Buffer<IsGlobal>::Buffer(const Schema &schema, const DataLayoutFactory &factory, bool load_simdfied,
2495 : : std::size_t num_tuples, setup_t setup, pipeline_t pipeline, teardown_t teardown)
2496 : 0 : : schema_(std::cref(schema))
2497 : 0 : , layout_(factory.make(schema, num_tuples))
2498 : 0 : , load_simdfied_(load_simdfied)
2499 : 0 : , setup_(std::move(setup))
2500 : 0 : , pipeline_(std::move(pipeline))
2501 : 0 : , teardown_(std::move(teardown))
2502 : : {
2503 [ # # # # : 0 : M_insist(schema.num_entries() != 0, "buffer schema must not be empty");
# # # # ]
2504 : :
2505 : : if constexpr (IsGlobal) {
2506 [ # # # # ]: 0 : if (layout_.is_finite()) {
2507 : : /*----- Pre-allocate memory for entire buffer. Use maximal possible alignment requirement of 8 bytes. ----*/
2508 [ # # ]: 0 : const uint32_t child_size_in_bytes = (layout_.stride_in_bits() + 7) / 8;
2509 : 0 : const uint32_t num_children =
2510 [ # # # # : 0 : (layout_.num_tuples() + layout_.child().num_tuples() - 1) / layout_.child().num_tuples();
# # # # #
# ]
2511 [ # # ]: 0 : storage_.base_address_ =
2512 [ # # # # ]: 0 : Module::Allocator().pre_allocate(num_children * child_size_in_bytes, /* alignment= */ 8);
2513 : 0 : } else {
2514 [ # # ]: 0 : storage_.capacity_.emplace(); // create global for capacity
2515 : : }
2516 : : }
2517 : 0 : }
2518 : :
2519 : : template<bool IsGlobal>
2520 : 0 : Buffer<IsGlobal>::~Buffer()
2521 : : {
2522 : : if constexpr (IsGlobal) { // free memory of global buffer when object is destroyed and no use may occur later
2523 [ # # # # ]: 0 : if (not layout_.is_finite()) {
2524 : : /*----- Deallocate memory for buffer. -----*/
2525 [ # # ]: 0 : M_insist(bool(storage_.capacity_));
2526 [ # # ]: 0 : const uint32_t child_size_in_bytes = (layout_.stride_in_bits() + 7) / 8;
2527 : : auto buffer_size_in_bytes =
2528 [ # # # # : 0 : (*storage_.capacity_ / uint32_t(layout_.child().num_tuples())) * child_size_in_bytes;
# # # # ]
2529 [ # # # # : 0 : Module::Allocator().deallocate(storage_.base_address_, buffer_size_in_bytes);
# # # # ]
2530 : 0 : }
2531 : : }
2532 : 0 : }
2533 : :
2534 : : template<bool IsGlobal>
2535 : 0 : buffer_load_proxy_t<IsGlobal> Buffer<IsGlobal>::create_load_proxy(param_t _tuple_value_schema,
2536 : : param_t _tuple_addr_schema) const
2537 : : {
2538 : : #ifndef NDEBUG
2539 [ # # # # ]: 0 : if (_tuple_value_schema) {
2540 [ # # # # ]: 0 : for (auto &e : _tuple_value_schema->get())
2541 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple value schema entry not found");
2542 : 0 : }
2543 [ # # # # ]: 0 : if (_tuple_addr_schema) {
2544 [ # # # # ]: 0 : for (auto &e : _tuple_addr_schema->get())
2545 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple address schema entry not found");
2546 : 0 : }
2547 : : #endif
2548 : :
2549 [ # # # # : 0 : static Schema empty_schema;
# # # # ]
2550 [ # # # # ]: 0 : const auto &tuple_value_schema = _tuple_value_schema ? _tuple_value_schema->get() : schema_.get();
2551 [ # # # # ]: 0 : const auto &tuple_addr_schema = _tuple_addr_schema ? _tuple_addr_schema->get() : empty_schema;
2552 : :
2553 : 0 : return buffer_load_proxy_t(*this, tuple_value_schema, tuple_addr_schema);
2554 : : }
2555 : :
2556 : : template<bool IsGlobal>
2557 : 0 : buffer_store_proxy_t<IsGlobal> Buffer<IsGlobal>::create_store_proxy(param_t tuple_schema) const
2558 : : {
2559 : : #ifndef NDEBUG
2560 [ # # # # ]: 0 : if (tuple_schema) {
2561 [ # # # # ]: 0 : for (auto &e : tuple_schema->get())
2562 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple schema entry not found");
2563 : 0 : }
2564 : : #endif
2565 : :
2566 [ # # # # ]: 0 : return tuple_schema ? buffer_store_proxy_t(*this, *tuple_schema) : buffer_store_proxy_t(*this, schema_);
2567 : : }
2568 : :
2569 : : template<bool IsGlobal>
2570 : 0 : buffer_swap_proxy_t<IsGlobal> Buffer<IsGlobal>::create_swap_proxy(param_t tuple_schema) const
2571 : : {
2572 : : #ifndef NDEBUG
2573 [ # # # # ]: 0 : if (tuple_schema) {
2574 [ # # # # ]: 0 : for (auto &e : tuple_schema->get())
2575 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple schema entry not found");
2576 : 0 : }
2577 : : #endif
2578 : :
2579 [ # # # # ]: 0 : return tuple_schema ? buffer_swap_proxy_t(*this, *tuple_schema) : buffer_swap_proxy_t(*this, schema_);
2580 : : }
2581 : :
2582 : : template<bool IsGlobal>
2583 : 0 : void Buffer<IsGlobal>::setup()
2584 : : {
2585 : 0 : M_insist(not base_address_, "must not call `setup()` twice");
2586 : 0 : M_insist(not size_, "must not call `setup()` twice");
2587 : 0 : M_insist(not capacity_, "must not call `setup()` twice");
2588 : 0 : M_insist(not first_iteration_, "must not call `setup()` twice");
2589 : :
2590 : : /*----- Create local variables. -----*/
2591 : 0 : base_address_.emplace();
2592 : 0 : size_.emplace();
2593 [ # # # # ]: 0 : if (not layout_.is_finite()) {
2594 : 0 : capacity_.emplace();
2595 : 0 : first_iteration_.emplace(true); // set to true
2596 : 0 : }
2597 : :
2598 : : /*----- For global buffers, read values from global backups into local variables. -----*/
2599 : : if constexpr (IsGlobal) {
2600 : : /* omit assigning base address here as it will always be set below */
2601 : 0 : *size_ = storage_.size_;
2602 [ # # ]: 0 : if (not layout_.is_finite()) {
2603 : 0 : M_insist(bool(storage_.capacity_));
2604 : 0 : *capacity_ = *storage_.capacity_;
2605 : 0 : }
2606 : : }
2607 : :
2608 [ # # # # ]: 0 : if (layout_.is_finite()) {
2609 : : if constexpr (IsGlobal) {
2610 : 0 : *base_address_ = storage_.base_address_; // buffer always already pre-allocated
2611 : : } else {
2612 : : /*----- Pre-allocate memory for entire buffer. Use maximal possible alignment requirement of 8 bytes. ----*/
2613 : 0 : const uint32_t child_size_in_bytes = (layout_.stride_in_bits() + 7) / 8;
2614 : 0 : const uint32_t num_children =
2615 : 0 : (layout_.num_tuples() + layout_.child().num_tuples() - 1) / layout_.child().num_tuples();
2616 [ # # ]: 0 : *base_address_ = Module::Allocator().pre_allocate(num_children * child_size_in_bytes, /* alignment= */ 8);
2617 : : }
2618 : 0 : } else {
2619 : : if constexpr (IsGlobal) {
2620 [ # # ]: 0 : IF (*capacity_ == 0U) { // buffer not yet allocated
2621 : : /*----- Set initial capacity. -----*/
2622 : 0 : *capacity_ = uint32_t(layout_.child().num_tuples());
2623 : :
2624 : : /*----- Allocate memory for one child instance. Use max. possible alignment requirement of 8 bytes. --*/
2625 : 0 : const uint32_t child_size_in_bytes = (layout_.stride_in_bits() + 7) / 8;
2626 [ # # ]: 0 : *base_address_ = Module::Allocator().allocate(child_size_in_bytes, /* alignment= */ 8);
2627 : 0 : } ELSE {
2628 : 0 : *base_address_ = storage_.base_address_;
2629 : 0 : };
2630 : : } else {
2631 : : /*----- Set initial capacity. -----*/
2632 : 0 : *capacity_ = uint32_t(layout_.child().num_tuples());
2633 : :
2634 : : /*----- Allocate memory for one child instance. Use max. possible alignment requirement of 8 bytes. -----*/
2635 : 0 : const uint32_t child_size_in_bytes = (layout_.stride_in_bits() + 7) / 8;
2636 [ # # ]: 0 : *base_address_ = Module::Allocator().allocate(child_size_in_bytes, /* alignment= */ 8);
2637 : : }
2638 : : }
2639 : 0 : }
2640 : :
2641 : : template<bool IsGlobal>
2642 : 0 : void Buffer<IsGlobal>::teardown()
2643 : : {
2644 : 0 : M_insist(bool(base_address_), "must call `setup()` before");
2645 : 0 : M_insist(bool(size_), "must call `setup()` before");
2646 : 0 : M_insist(not layout_.is_finite() == bool(capacity_), "must call `setup()` before");
2647 : 0 : M_insist(not layout_.is_finite() == bool(first_iteration_), "must call `setup()` before");
2648 : :
2649 : : if constexpr (not IsGlobal) { // free memory of local buffer when user calls teardown method
2650 [ # # ]: 0 : if (not layout_.is_finite()) {
2651 : : /*----- Deallocate memory for buffer. -----*/
2652 : 0 : const uint32_t child_size_in_bytes = (layout_.stride_in_bits() + 7) / 8;
2653 [ # # ]: 0 : auto buffer_size_in_bytes = (*capacity_ / uint32_t(layout_.child().num_tuples())) * child_size_in_bytes;
2654 [ # # # # : 0 : Module::Allocator().deallocate(*base_address_, buffer_size_in_bytes);
# # # # ]
2655 : 0 : }
2656 : : }
2657 : :
2658 : : /*----- For global buffers, write values from local variables into global backups. -----*/
2659 : : if constexpr (IsGlobal) {
2660 : 0 : storage_.base_address_ = *base_address_;
2661 : 0 : storage_.size_ = *size_;
2662 [ # # ]: 0 : if (not layout_.is_finite()) {
2663 : 0 : M_insist(bool(storage_.capacity_));
2664 : 0 : *storage_.capacity_ = *capacity_;
2665 : 0 : }
2666 : : }
2667 : :
2668 : : /*----- Destroy local variables. -----*/
2669 : 0 : base_address_.reset();
2670 : 0 : size_.reset();
2671 [ # # # # ]: 0 : if (not layout_.is_finite()) {
2672 : 0 : capacity_.reset();
2673 [ # # # # ]: 0 : first_iteration_->val().discard(); // artificial use to silence diagnostics if `consume()` is not called
2674 : 0 : first_iteration_.reset();
2675 : 0 : }
2676 : 0 : }
2677 : :
2678 : : template<bool IsGlobal>
2679 : 0 : void Buffer<IsGlobal>::resume_pipeline(param_t _tuple_value_schema, param_t _tuple_addr_schema) const
2680 : : {
2681 [ # # # # ]: 0 : if (not pipeline_)
2682 : 0 : return;
2683 : :
2684 [ # # # # : 0 : static Schema empty_schema;
# # # # ]
2685 [ # # # # ]: 0 : const auto &tuple_value_schema = _tuple_value_schema ? _tuple_value_schema->get() : schema_.get();
2686 [ # # # # ]: 0 : const auto &tuple_addr_schema = _tuple_addr_schema ? _tuple_addr_schema->get() : empty_schema;
2687 : :
2688 : : #ifndef NDEBUG
2689 [ # # # # ]: 0 : for (auto &e : tuple_value_schema)
2690 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple value schema entry not found");
2691 [ # # # # ]: 0 : for (auto &e : tuple_addr_schema)
2692 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple addr schema entry not found");
2693 : : #endif
2694 : :
2695 : : /*----- Create function on-demand to assert that all needed identifiers are already created. -----*/
2696 [ # # # # ]: 0 : if (not resume_pipeline_) {
2697 : : /*----- Create function to resume the pipeline for each tuple contained in the buffer. -----*/
2698 [ # # # # : 0 : FUNCTION(resume_pipeline, void(void*, uint32_t))
# # # # #
# # # ]
2699 : : {
2700 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # ]
2701 : :
2702 : : /*----- Access base address and size parameters. -----*/
2703 [ # # # # : 0 : Ptr<void> base_address = PARAMETER(0);
# # # # ]
2704 [ # # # # : 0 : U32x1 size = PARAMETER(1);
# # # # ]
2705 : :
2706 : : /*----- Compute poss. number of SIMD lanes and decide which to use with regard to other ops. preferences. */
2707 : 0 : const auto num_simd_lanes_preferred =
2708 [ # # # # : 0 : CodeGenContext::Get().num_simd_lanes_preferred(); // get other operators preferences
# # # # ]
2709 : 0 : const std::size_t num_simd_lanes =
2710 [ # # # # : 0 : load_simdfied_ ? std::max<std::size_t>({ num_simd_lanes_preferred,
# # # # ]
2711 [ # # # # ]: 0 : get_num_simd_lanes(layout_, schema_, tuple_value_schema),
2712 [ # # # # ]: 0 : tuple_addr_schema.empty() ? 0UL : 4UL }) // 32-bit pointers and 128-bit SIMD vectors
2713 : : : 1;
2714 [ # # # # : 0 : CodeGenContext::Get().set_num_simd_lanes(num_simd_lanes);
# # # # ]
2715 : :
2716 : : /*----- Emit setup code *before* compiling data layout to not overwrite its temporary boolean variables. -*/
2717 [ # # # # ]: 0 : setup_();
2718 : :
2719 [ # # # # ]: 0 : Var<U32x1> load_tuple_id; // default initialized to 0
2720 : :
2721 [ # # # # : 0 : if (tuple_value_schema.num_entries() == 0 and tuple_addr_schema.num_entries() == 0) {
# # # # #
# # # # #
# # ]
2722 : : /*----- If no attributes must be loaded, generate a loop just executing the pipeline `size`-times. -----*/
2723 [ # # # # : 0 : WHILE (load_tuple_id < size) {
# # # # #
# # # # #
# # ]
2724 [ # # # # ]: 0 : load_tuple_id += uint32_t(num_simd_lanes);
2725 [ # # # # ]: 0 : pipeline_();
2726 : : }
2727 [ # # # # ]: 0 : base_address.discard(); // since it is not needed
2728 : 0 : } else {
2729 : : /*----- Compile data layout to generate sequential load from buffer. -----*/
2730 : 0 : auto [load_inits, loads, load_jumps] =
2731 [ # # # # : 0 : compile_load_sequential(tuple_value_schema, tuple_addr_schema, base_address, layout_,
# # # # #
# # # ]
2732 : 0 : num_simd_lanes, schema_, load_tuple_id);
2733 : :
2734 : : /*----- Generate loop for loading entire buffer, with the pipeline emitted into the loop body. -----*/
2735 [ # # # # ]: 0 : load_inits.attach_to_current();
2736 [ # # # # : 0 : WHILE (load_tuple_id < size) {
# # # # #
# # # # #
# # ]
2737 [ # # # # ]: 0 : loads.attach_to_current();
2738 [ # # # # ]: 0 : pipeline_();
2739 [ # # # # ]: 0 : load_jumps.attach_to_current();
2740 : : }
2741 : 0 : }
2742 : :
2743 : : /*----- Emit teardown code. -----*/
2744 [ # # # # ]: 0 : teardown_();
2745 : 0 : }
2746 : 0 : resume_pipeline_ = std::move(resume_pipeline);
2747 : 0 : }
2748 : :
2749 : : /*----- Call created function. -----*/
2750 : 0 : M_insist(bool(resume_pipeline_));
2751 [ # # # # : 0 : (*resume_pipeline_)(base_address(), size()); // base address and size as arguments
# # # # ]
2752 : 0 : }
2753 : :
2754 : : template<bool IsGlobal>
2755 : 0 : void Buffer<IsGlobal>::resume_pipeline_inline(param_t tuple_value_schema, param_t tuple_addr_schema) const
2756 : : {
2757 [ # # # # : 0 : execute_pipeline_inline(setup_, pipeline_, teardown_, std::move(tuple_value_schema), std::move(tuple_addr_schema));
# # # # #
# # # ]
2758 : 0 : }
2759 : :
2760 : : template<bool IsGlobal>
2761 : 0 : void Buffer<IsGlobal>::execute_pipeline(setup_t setup, pipeline_t pipeline, teardown_t teardown,
2762 : : param_t _tuple_value_schema, param_t _tuple_addr_schema) const
2763 : : {
2764 [ # # # # ]: 0 : if (not pipeline)
2765 : 0 : return;
2766 : :
2767 [ # # # # : 0 : static Schema empty_schema;
# # # # ]
2768 [ # # # # ]: 0 : const auto &tuple_value_schema = _tuple_value_schema ? _tuple_value_schema->get() : schema_.get();
2769 [ # # # # ]: 0 : const auto &tuple_addr_schema = _tuple_addr_schema ? _tuple_addr_schema->get() : empty_schema;
2770 : :
2771 : : #ifndef NDEBUG
2772 [ # # # # ]: 0 : for (auto &e : tuple_value_schema)
2773 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple value schema entry not found");
2774 [ # # # # ]: 0 : for (auto &e : tuple_addr_schema)
2775 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple addr schema entry not found");
2776 : : #endif
2777 : :
2778 : : /*----- Create function to resume the pipeline for each tuple contained in the buffer. -----*/
2779 [ # # # # : 0 : FUNCTION(resume_pipeline, void(void*, uint32_t))
# # # # #
# # # ]
2780 : : {
2781 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # ]
2782 : :
2783 : : /*----- Access base address and size parameters. -----*/
2784 [ # # # # : 0 : Ptr<void> base_address = PARAMETER(0);
# # # # ]
2785 [ # # # # : 0 : U32x1 size = PARAMETER(1);
# # # # ]
2786 : :
2787 : : /*----- Compute poss. number of SIMD lanes and decide which to use with regard to other ops. preferences. */
2788 : 0 : const auto num_simd_lanes_preferred =
2789 [ # # # # : 0 : CodeGenContext::Get().num_simd_lanes_preferred(); // get other operators preferences
# # # # ]
2790 : 0 : const std::size_t num_simd_lanes =
2791 [ # # # # : 0 : load_simdfied_ ? std::max<std::size_t>({ num_simd_lanes_preferred,
# # # # ]
2792 [ # # # # ]: 0 : get_num_simd_lanes(layout_, schema_, tuple_value_schema),
2793 [ # # # # ]: 0 : tuple_addr_schema.empty() ? 0UL : 4UL }) // 32-bit pointers and 128-bit SIMD vectors
2794 : : : 1;
2795 [ # # # # : 0 : CodeGenContext::Get().set_num_simd_lanes(num_simd_lanes);
# # # # ]
2796 : :
2797 : : /*----- Emit setup code *before* compiling data layout to not overwrite its temporary boolean variables. -*/
2798 [ # # # # ]: 0 : setup();
2799 : :
2800 [ # # # # ]: 0 : Var<U32x1> load_tuple_id; // default initialized to 0
2801 : :
2802 [ # # # # : 0 : if (tuple_value_schema.num_entries() == 0 and tuple_addr_schema.num_entries() == 0) {
# # # # #
# # # # #
# # ]
2803 : : /*----- If no attributes must be loaded, generate a loop just executing the pipeline `size`-times. -----*/
2804 [ # # # # : 0 : WHILE (load_tuple_id < size) {
# # # # #
# # # # #
# # ]
2805 [ # # # # ]: 0 : load_tuple_id += uint32_t(num_simd_lanes);
2806 [ # # # # ]: 0 : pipeline();
2807 : : }
2808 [ # # # # ]: 0 : base_address.discard(); // since it is not needed
2809 : 0 : } else {
2810 : : /*----- Compile data layout to generate sequential load from buffer. -----*/
2811 : 0 : auto [load_inits, loads, load_jumps] =
2812 [ # # # # : 0 : compile_load_sequential(tuple_value_schema, tuple_addr_schema, base_address, layout_, num_simd_lanes,
# # # # #
# # # ]
2813 : 0 : schema_, load_tuple_id);
2814 : :
2815 : : /*----- Generate loop for loading entire buffer, with the pipeline emitted into the loop body. -----*/
2816 [ # # # # ]: 0 : load_inits.attach_to_current();
2817 [ # # # # : 0 : WHILE (load_tuple_id < size) {
# # # # #
# # # # #
# # ]
2818 [ # # # # ]: 0 : loads.attach_to_current();
2819 [ # # # # ]: 0 : pipeline();
2820 [ # # # # ]: 0 : load_jumps.attach_to_current();
2821 : : }
2822 : 0 : }
2823 : :
2824 : : /*----- Emit teardown code. -----*/
2825 [ # # # # ]: 0 : teardown();
2826 : 0 : }
2827 : :
2828 : : /*----- Call created function. -----*/
2829 [ # # # # : 0 : resume_pipeline(base_address(), size()); // base address and size as arguments
# # # # #
# # # ]
2830 : 0 : }
2831 : :
2832 : : template<bool IsGlobal>
2833 : 0 : void Buffer<IsGlobal>::execute_pipeline_inline(setup_t setup, pipeline_t pipeline, teardown_t teardown,
2834 : : param_t _tuple_value_schema, param_t _tuple_addr_schema) const
2835 : : {
2836 [ # # # # ]: 0 : if (not pipeline)
2837 : 0 : return;
2838 : :
2839 [ # # # # : 0 : static Schema empty_schema;
# # # # ]
2840 [ # # # # ]: 0 : const auto &tuple_value_schema = _tuple_value_schema ? _tuple_value_schema->get() : schema_.get();
2841 [ # # # # ]: 0 : const auto &tuple_addr_schema = _tuple_addr_schema ? _tuple_addr_schema->get() : empty_schema;
2842 : :
2843 : : #ifndef NDEBUG
2844 [ # # # # ]: 0 : for (auto &e : tuple_value_schema)
2845 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple value schema entry not found");
2846 [ # # # # ]: 0 : for (auto &e : tuple_addr_schema)
2847 : 0 : M_insist(schema_.get().find(e.id) != schema_.get().cend(), "tuple addr schema entry not found");
2848 : : #endif
2849 : :
2850 : : /*----- Access base address and size depending on whether they are globals or locals. -----*/
2851 : : Ptr<void> base_address =
2852 [ # # # # : 0 : M_CONSTEXPR_COND(IsGlobal,
# # # # #
# # # #
# ]
2853 : : base_address_ ? base_address_->val() : Var<Ptr<void>>(storage_.base_address_.val()).val(),
2854 : : ({ M_insist(bool(base_address_)); base_address_->val(); }));
2855 : : U32x1 size =
2856 [ # # # # : 0 : M_CONSTEXPR_COND(IsGlobal,
# # # # #
# # # # #
# # # # ]
2857 : : size_ ? size_->val() : Var<U32x1>(storage_.size_.val()).val(),
2858 : : ({ M_insist(bool(size_)); size_->val(); }));
2859 : :
2860 : : /*----- If predication is used, compute number of tuples to load from buffer depending on predicate. -----*/
2861 : 0 : std::optional<Var<Boolx1>> pred; // use variable since WHILE loop will clone it (for IF and DO_WHILE)
2862 [ # # # # : 0 : if (auto &env = CodeGenContext::Get().env(); env.predicated()) {
# # # # #
# # # # #
# # ]
2863 [ # # # # : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
# # # # #
# # # ]
2864 [ # # # # : 0 : pred = env.extract_predicate<_Boolx1>().is_true_and_not_null();
# # # # #
# # # ]
2865 : 0 : }
2866 [ # # # # : 0 : U32x1 num_tuples = pred ? Select(*pred, size, 0U) : size;
# # # # #
# # # ]
2867 : :
2868 : : /*----- Compute possible number of SIMD lanes and decide which to use with regard to other operators preferences. */
2869 : 0 : const auto num_simd_lanes_preferred =
2870 [ # # # # : 0 : CodeGenContext::Get().num_simd_lanes_preferred(); // get other operators preferences
# # # # ]
2871 : 0 : const std::size_t num_simd_lanes =
2872 [ # # # # : 0 : load_simdfied_ ? std::max<std::size_t>({ num_simd_lanes_preferred,
# # # # ]
2873 [ # # # # ]: 0 : get_num_simd_lanes(layout_, schema_, tuple_value_schema),
2874 [ # # # # ]: 0 : tuple_addr_schema.empty() ? 0UL : 4UL }) // 32-bit pointers and 128-bit SIMD vectors
2875 : : : 1;
2876 [ # # # # : 0 : CodeGenContext::Get().set_num_simd_lanes(num_simd_lanes);
# # # # ]
2877 : :
2878 : : /*----- Emit setup code *before* compiling data layout to not overwrite its temporary boolean variables. -----*/
2879 [ # # # # ]: 0 : setup();
2880 : :
2881 [ # # # # ]: 0 : Var<U32x1> load_tuple_id(0); // explicitly (re-)set tuple ID to 0
2882 : :
2883 [ # # # # : 0 : if (tuple_value_schema.num_entries() == 0 and tuple_addr_schema.num_entries() == 0) {
# # # # #
# # # # #
# # ]
2884 : : /*----- If no attributes must be loaded, generate a loop just executing the pipeline `size`-times. -----*/
2885 [ # # # # : 0 : WHILE (load_tuple_id < num_tuples) {
# # # # #
# # # # #
# # ]
2886 [ # # # # ]: 0 : load_tuple_id += uint32_t(num_simd_lanes);
2887 [ # # # # ]: 0 : pipeline();
2888 : : }
2889 [ # # # # ]: 0 : base_address.discard(); // since it is not needed
2890 : 0 : } else {
2891 : : /*----- Compile data layout to generate sequential load from buffer. -----*/
2892 : 0 : auto [load_inits, loads, load_jumps] =
2893 [ # # # # : 0 : compile_load_sequential(tuple_value_schema, tuple_addr_schema, base_address, layout_,
# # # # #
# # # ]
2894 : 0 : num_simd_lanes, schema_, load_tuple_id);
2895 : :
2896 : : /*----- Generate loop for loading entire buffer, with the pipeline emitted into the loop body. -----*/
2897 [ # # # # ]: 0 : load_inits.attach_to_current();
2898 [ # # # # : 0 : WHILE (load_tuple_id < num_tuples) {
# # # # #
# # # # #
# # ]
2899 [ # # # # ]: 0 : loads.attach_to_current();
2900 [ # # # # ]: 0 : pipeline();
2901 [ # # # # ]: 0 : load_jumps.attach_to_current();
2902 : : }
2903 : 0 : }
2904 : :
2905 : : /*----- Emit teardown code. -----*/
2906 [ # # # # ]: 0 : teardown();
2907 : 0 : }
2908 : :
2909 : : template<bool IsGlobal>
2910 : 0 : void Buffer<IsGlobal>::consume()
2911 : : {
2912 : 0 : M_insist(bool(base_address_), "must call `setup()` before");
2913 : 0 : M_insist(bool(size_), "must call `setup()` before");
2914 : 0 : M_insist(not layout_.is_finite() == bool(capacity_), "must call `setup()` before");
2915 : 0 : M_insist(not layout_.is_finite() == bool(first_iteration_), "must call `setup()` before");
2916 : :
2917 : : /*----- Compile data layout to generate sequential single-pass store into the buffer. -----*/
2918 : : /* We are able to use a single-pass store, i.e. *local* pointers and masks, since we explicitly save the needed
2919 : : * variables, i.e. base address and size, using *global* backups and restore them before performing the actual
2920 : : * store in the case of global buffers. For local buffers, stores must be done in a single pass anyway. */
2921 [ # # # # : 0 : static Schema empty_schema;
# # # # ]
2922 : 0 : auto [_store_inits, stores, _store_jumps] =
2923 [ # # # # ]: 0 : compile_store_sequential_single_pass(schema_, empty_schema, *base_address_, layout_,
2924 [ # # # # : 0 : CodeGenContext::Get().num_simd_lanes(), schema_, *size_);
# # # # ]
2925 [ # # # # : 0 : Block store_inits(std::move(_store_inits)), store_jumps(std::move(_store_jumps));
# # # # ]
2926 : :
2927 [ # # # # : 0 : if (layout_.is_finite()) {
# # # # ]
2928 [ # # # # : 0 : IF (*size_ == 0U) { // buffer empty
# # # # ]
2929 : : /*----- Emit initialization code for storing (i.e. (re-)set to first buffer slot). -----*/
2930 : 0 : store_inits.attach_to_current();
2931 : 0 : };
2932 : 0 : } else {
2933 [ # # # # : 0 : IF (*size_ == *capacity_) { // buffer full
# # # # ]
2934 : : /*----- Resize buffer by doubling its capacity. -----*/
2935 : 0 : const uint32_t child_size_in_bytes = (layout_.stride_in_bits() + 7) / 8;
2936 [ # # # # ]: 0 : auto buffer_size_in_bytes = (*capacity_ / uint32_t(layout_.child().num_tuples())) * child_size_in_bytes;
2937 [ # # # # : 0 : auto ptr = Module::Allocator().allocate(buffer_size_in_bytes.clone());
# # # # #
# # # ]
2938 [ # # # # : 0 : Wasm_insist(ptr == *base_address_ + buffer_size_in_bytes.make_signed(),
# # # # #
# # # # #
# # # # #
# ]
2939 : : "buffer could not be resized sequentially in memory");
2940 [ # # # # ]: 0 : *capacity_ *= 2U;
2941 : 0 : };
2942 : :
2943 [ # # # # ]: 0 : IF (*first_iteration_) {
2944 : : /*----- Emit initialization code for storing (i.e. set to current buffer slot). -----*/
2945 : 0 : store_inits.attach_to_current();
2946 : :
2947 : 0 : *first_iteration_ = false;
2948 : 0 : };
2949 : : }
2950 : :
2951 : : /*----- Emit storing code. -----*/
2952 [ # # # # ]: 0 : stores.attach_to_current();
2953 : :
2954 [ # # # # : 0 : if (layout_.is_finite()) {
# # # # ]
2955 [ # # # # : 0 : IF (*size_ == uint32_t(layout_.num_tuples() - CodeGenContext::Get().num_simd_lanes())) { // buffer full
# # # # #
# # # # #
# # # # #
# ]
2956 : : /*----- Resume pipeline for each tuple in buffer and reset size of buffer to 0. -----*/
2957 : 0 : *size_ = uint32_t(layout_.num_tuples()); // increment size of buffer to resume pipeline even for last tuple
2958 : 0 : resume_pipeline();
2959 : 0 : *size_ = 0U;
2960 : 0 : } ELSE { // buffer not full
2961 : : /*----- Emit advancing code to next buffer slot and increment size of buffer. -----*/
2962 : 0 : store_jumps.attach_to_current();
2963 : 0 : };
2964 : 0 : } else {
2965 : : /*----- Emit advancing code to next buffer slot and increment size of buffer. -----*/
2966 [ # # # # ]: 0 : store_jumps.attach_to_current();
2967 : : }
2968 : 0 : }
2969 : :
2970 : : // explicit instantiations to prevent linker errors
2971 : : template struct m::wasm::Buffer<false>;
2972 : : template struct m::wasm::Buffer<true>;
2973 : :
2974 : :
2975 : : /*======================================================================================================================
2976 : : * buffer accesses
2977 : : *====================================================================================================================*/
2978 : :
2979 : : template<bool IsGlobal>
2980 : 0 : void buffer_swap_proxy_t<IsGlobal>::operator()(U32x1 first, U32x1 second)
2981 : : {
2982 : : /*----- Create load proxy. -----*/
2983 : 0 : auto load = buffer_.get().create_load_proxy(schema_.get());
2984 : :
2985 : : /*----- Load first tuple into fresh environment. -----*/
2986 : 0 : auto env_first = [&](){
2987 : 0 : auto S = CodeGenContext::Get().scoped_environment();
2988 [ # # # # : 0 : load(first.clone());
# # # # ]
2989 [ # # # # ]: 0 : return S.extract();
2990 : 0 : }();
2991 : :
2992 [ # # # # : 0 : operator()(first, second, env_first);
# # # # #
# # # ]
2993 : 0 : }
2994 : :
2995 : : template<bool IsGlobal>
2996 : 0 : void buffer_swap_proxy_t<IsGlobal>::operator()(U32x1 first, U32x1 second, const Environment &env_first)
2997 : : {
2998 : : /*----- Create load and store proxies. -----*/
2999 : 0 : auto load = buffer_.get().create_load_proxy(schema_.get());
3000 : 0 : auto store = buffer_.get().create_store_proxy(schema_.get());
3001 : :
3002 : : /*----- Temporarily save first tuple by creating variable or separate string buffer. -----*/
3003 : 0 : Environment _env_first;
3004 [ # # # # : 0 : for (auto &e : schema_.get()) {
# # # # #
# # # ]
3005 [ # # # # ]: 0 : std::visit(overloaded {
3006 : 0 : [&](NChar value) -> void {
3007 : 0 : Var<Ptr<Charx1>> ptr; // always set here
3008 [ # # # # : 0 : IF (value.clone().is_null()) {
# # # # #
# # # ]
3009 [ # # # # ]: 0 : ptr = Ptr<Charx1>::Nullptr();
3010 : 0 : } ELSE {
3011 [ # # # # ]: 0 : ptr = Module::Allocator().pre_malloc<char>(value.size_in_bytes());
3012 [ # # # # : 0 : strncpy(ptr, value, U32x1(value.size_in_bytes())).discard();
# # # # #
# # # # #
# # # # #
# ]
3013 : 0 : };
3014 [ # # # # : 0 : _env_first.add(e.id, NChar(ptr, value.can_be_null(), value.length(), value.guarantees_terminating_nul()));
# # # # #
# # # # #
# # ]
3015 : 0 : },
3016 : 0 : [&]<typename T>(Expr<T> value) -> void {
3017 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3018 : 0 : Var<Expr<T>> var(value);
3019 [ # # # # : 0 : _env_first.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3020 : 0 : } else {
3021 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(value.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3022 [ # # # # : 0 : _env_first.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3023 : 0 : }
3024 : 0 : },
3025 : 0 : [](auto) -> void { M_unreachable("SIMDfication currently not supported"); },
3026 : 0 : [](std::monostate) -> void { M_unreachable("value must be loaded beforehand"); },
3027 [ # # # # ]: 0 : }, env_first.get(e.id));
3028 : : }
3029 : :
3030 : : /*----- Load second tuple in scoped environment and store it directly at first tuples address. -----*/
3031 : : {
3032 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment();
# # # # ]
3033 [ # # # # : 0 : load(second.clone());
# # # # ]
3034 [ # # # # : 0 : store(first);
# # # # ]
3035 : 0 : }
3036 : :
3037 : : /*----- Store temporarily saved first tuple at second tuples address. ----*/
3038 : : {
3039 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(std::move(_env_first));
# # # # #
# # # ]
3040 [ # # # # : 0 : store(second);
# # # # ]
3041 : 0 : }
3042 : 0 : }
3043 : :
3044 : : template<bool IsGlobal>
3045 : 0 : void buffer_swap_proxy_t<IsGlobal>::operator()(U32x1 first, U32x1 second, const Environment &env_first,
3046 : : const Environment &env_second)
3047 : : {
3048 : : /*----- Create store proxy. -----*/
3049 : 0 : auto store = buffer_.get().create_store_proxy(schema_.get());
3050 : :
3051 : : /*----- Temporarily save first tuple by creating variable or separate string buffer. -----*/
3052 : 0 : Environment _env_first;
3053 [ # # # # : 0 : for (auto &e : schema_.get()) {
# # # # #
# # # ]
3054 [ # # # # ]: 0 : std::visit(overloaded {
3055 : 0 : [&](NChar value) -> void {
3056 : 0 : Var<Ptr<Charx1>> ptr; // always set here
3057 [ # # # # : 0 : IF (value.clone().is_null()) {
# # # # #
# # # ]
3058 [ # # # # ]: 0 : ptr = Ptr<Charx1>::Nullptr();
3059 : 0 : } ELSE {
3060 [ # # # # ]: 0 : ptr = Module::Allocator().pre_malloc<char>(value.size_in_bytes());
3061 [ # # # # : 0 : strncpy(ptr, value, U32x1(value.size_in_bytes())).discard();
# # # # #
# # # # #
# # # # #
# ]
3062 : 0 : };
3063 [ # # # # : 0 : _env_first.add(e.id, NChar(ptr, value.can_be_null(), value.length(), value.guarantees_terminating_nul()));
# # # # #
# # # # #
# # ]
3064 : 0 : },
3065 : 0 : [&]<typename T>(Expr<T> value) -> void {
3066 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3067 : 0 : Var<Expr<T>> var(value);
3068 [ # # # # : 0 : _env_first.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3069 : 0 : } else {
3070 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(value.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3071 [ # # # # : 0 : _env_first.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3072 : 0 : }
3073 : 0 : },
3074 : 0 : [](auto) -> void { M_unreachable("SIMDfication currently not supported"); },
3075 : 0 : [](std::monostate) -> void { M_unreachable("value must be loaded beforehand"); }
3076 [ # # # # ]: 0 : }, env_first.get(e.id));
3077 : : }
3078 : :
3079 : : /*----- Store already loaded second tuple directly at first tuples address. -----*/
3080 : : {
3081 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment();
# # # # ]
3082 [ # # # # : 0 : CodeGenContext::Get().env().add(env_second);
# # # # #
# # # ]
3083 [ # # # # : 0 : store(first);
# # # # ]
3084 : 0 : }
3085 : :
3086 : : /*----- Store temporarily saved first tuple at second tuples address. ----*/
3087 : : {
3088 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(std::move(_env_first));
# # # # #
# # # ]
3089 [ # # # # : 0 : store(second);
# # # # ]
3090 : 0 : }
3091 : 0 : }
3092 : :
3093 : : // explicit instantiations to prevent linker errors
3094 : : template struct m::wasm::buffer_swap_proxy_t<false>;
3095 : : template struct m::wasm::buffer_swap_proxy_t<true>;
3096 : :
3097 : :
3098 : : /*======================================================================================================================
3099 : : * string comparison
3100 : : *====================================================================================================================*/
3101 : :
3102 : 54 : _I32x1 m::wasm::strncmp(NChar _left, NChar _right, U32x1 len, bool reverse)
3103 : : {
3104 : : static thread_local struct {} _; // unique caller handle
3105 : : struct data_t : GarbageCollectedData
3106 : : {
3107 : : public:
3108 : : using fn_t = int32_t(uint32_t, uint32_t, char*, char*, uint32_t);
3109 : : std::optional<FunctionProxy<fn_t>> strncmp_terminating_nul;
3110 : : std::optional<FunctionProxy<fn_t>> strncmp_no_terminating_nul;
3111 : :
3112 : 54 : data_t(GarbageCollectedData &&d) : GarbageCollectedData(std::move(d)) { }
3113 : : };
3114 : 54 : auto &d = Module::Get().add_garbage_collected_data<data_t>(&_); // garbage collect the `data_t` instance
3115 : :
3116 : 108 : auto strncmp_non_null = [&d, &_left, &_right, &reverse](Ptr<Charx1> left, Ptr<Charx1> right, U32x1 len) -> I32x1 {
3117 [ + - + - ]: 54 : Wasm_insist(left.clone().not_null(), "left operand must not be NULL");
3118 [ + - + - ]: 54 : Wasm_insist(right.clone().not_null(), "right operand must not be NULL");
3119 [ + - + - ]: 54 : Wasm_insist(len.clone() != 0U, "length to compare must not be 0");
3120 : :
3121 [ - + # # ]: 54 : if (_left.length() == 1 and _right.length() == 1) {
3122 : : /*----- Special handling of single char strings. -----*/
3123 : 0 : len.discard();
3124 [ # # # # : 0 : auto left_gt_right = *left.clone() > *right.clone();
# # # # ]
3125 [ # # # # : 0 : return left_gt_right.to<int32_t>() - (*left < *right).to<int32_t>();
# # # # #
# # # ]
3126 : 0 : } else {
3127 [ + + + - : 54 : if (_left.guarantees_terminating_nul() and _right.guarantees_terminating_nul() and not reverse) { // reverse needs in-bounds checks
- + ]
3128 [ - + ]: 18 : if (not d.strncmp_terminating_nul) {
3129 : : /*----- Create function to compute the result for non-nullptr arguments character-wise. -----*/
3130 [ + - + - : 36 : FUNCTION(strncmp_terminating_nul, data_t::fn_t)
+ - ]
3131 : : {
3132 [ + - + - ]: 18 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
3133 : :
3134 [ + - ]: 18 : const auto len_ty_left = PARAMETER(0);
3135 [ + - ]: 18 : const auto len_ty_right = PARAMETER(1);
3136 [ + - ]: 18 : auto left = PARAMETER(2);
3137 [ + - ]: 18 : auto right = PARAMETER(3);
3138 [ + - ]: 18 : const auto len = PARAMETER(4);
3139 : :
3140 [ + - ]: 18 : Var<I32x1> result; // always set here
3141 : :
3142 [ + - + - : 18 : I32x1 len_left = Select(len < len_ty_left, len, len_ty_left) .make_signed();
+ - ]
3143 [ + - + - : 18 : I32x1 len_right = Select(len < len_ty_right, len, len_ty_right).make_signed();
+ - ]
3144 [ + - + - ]: 18 : Var<Ptr<Charx1>> end_left (left + len_left);
3145 [ + - + - ]: 18 : Var<Ptr<Charx1>> end_right(right + len_right);
3146 : :
3147 [ + - + - ]: 36 : LOOP() {
3148 : : /* Check whether one side is shorter than the other. */
3149 [ + - + - : 18 : result = (left != end_left).to<int32_t>() - (right != end_right).to<int32_t>();
+ - + - +
- + - ]
3150 [ + - + - : 18 : BREAK(result != 0 or left == end_left); // at the end of either or both strings
+ - + - ]
3151 : :
3152 : : /* Compare by current character. Loading is valid since we have not seen the terminating
3153 : : * NUL byte yet. */
3154 [ + - + - : 18 : result = (*left > *right).to<int32_t>() - (*left < *right).to<int32_t>();
+ - + - +
- + - + -
+ - + - +
- ]
3155 [ + - + - ]: 18 : BREAK(result != 0); // found first position where strings differ
3156 [ + - + - : 18 : BREAK(*left == 0); // reached end of identical strings
+ - ]
3157 : :
3158 : : /* Advance to next character. */
3159 [ + - ]: 18 : left += 1;
3160 [ + - ]: 18 : right += 1;
3161 [ + - ]: 18 : CONTINUE();
3162 : : }
3163 : :
3164 [ + - ]: 18 : RETURN(result);
3165 : 18 : }
3166 : 18 : d.strncmp_terminating_nul = std::move(strncmp_terminating_nul);
3167 : 18 : }
3168 : :
3169 : : /*----- Call strncmp_terminating_nul function. ------*/
3170 : 18 : M_insist(bool(d.strncmp_terminating_nul));
3171 : 18 : return (*d.strncmp_terminating_nul)(_left.length(), _right.length(), left, right, len);
3172 : : } else {
3173 [ - + ]: 36 : if (not d.strncmp_no_terminating_nul) {
3174 : : /*----- Create function to compute the result for non-nullptr arguments character-wise. -----*/
3175 [ + - + - : 72 : FUNCTION(strncmp_no_terminating_nul, data_t::fn_t)
+ - ]
3176 : : {
3177 [ + - + - ]: 36 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
3178 : :
3179 [ + - ]: 36 : const auto len_ty_left = PARAMETER(0);
3180 [ + - ]: 36 : const auto len_ty_right = PARAMETER(1);
3181 [ + - + - ]: 36 : Var<Ptr<Charx1>> left(PARAMETER(2));
3182 [ + - + - ]: 36 : Var<Ptr<Charx1>> right(PARAMETER(3));
3183 [ + - ]: 36 : const auto len = PARAMETER(4);
3184 : :
3185 [ + - ]: 36 : Var<I32x1> result; // always set here
3186 : :
3187 [ + - + - : 36 : I32x1 len_left = Select(len < len_ty_left, len, len_ty_left) .make_signed();
+ - ]
3188 [ + - + - : 36 : I32x1 len_right = Select(len < len_ty_right, len, len_ty_right).make_signed();
+ - ]
3189 [ + - + - ]: 36 : Var<Ptr<Charx1>> end_left, end_right;
3190 : :
3191 [ + - ]: 36 : if (not reverse) {
3192 : : /* Set end variables according to theoretical length. */
3193 [ + - + - ]: 36 : end_left = left + len_left;
3194 [ + - + - ]: 36 : end_right = right + len_right;
3195 : 36 : } else {
3196 : : /* Set end variables to first found NUL byte without exceeding the theoretical length. */
3197 [ # # ]: 0 : end_left = left;
3198 [ # # # # : 0 : WHILE(*end_left != 0 and end_left != left + len_left) {
# # # # #
# # # # #
# # ]
3199 [ # # ]: 0 : end_left += 1;
3200 : : }
3201 [ # # ]: 0 : end_right = right;
3202 [ # # # # : 0 : WHILE(*end_right != 0 and end_right != right + len_right) {
# # # # #
# # # # #
# # ]
3203 [ # # ]: 0 : end_right += 1;
3204 : : }
3205 : :
3206 : : /* Swap variable for current position with the one for end position to iterate reversed. */
3207 [ # # ]: 0 : swap(left, end_left);
3208 [ # # ]: 0 : swap(right, end_right);
3209 : :
3210 : : /* Resolve off-by-one errors created by swapping variables. */
3211 [ # # ]: 0 : left -= 1;
3212 [ # # ]: 0 : right -= 1;
3213 [ # # ]: 0 : end_left -= 1;
3214 [ # # ]: 0 : end_right -= 1;
3215 : : }
3216 : :
3217 [ + - + - ]: 72 : LOOP() {
3218 : : /* Check whether one side is shorter than the other. Load next character with in-bounds
3219 : : * checks since the strings may not be NUL byte terminated. */
3220 [ + - + - ]: 36 : Var<Charx1> val_left, val_right;
3221 [ + - + - ]: 72 : IF (left != end_left) {
3222 [ + - ]: 36 : val_left = *left;
3223 : 108 : } ELSE {
3224 : 36 : val_left = '\0';
3225 : 36 : };
3226 [ + - + - ]: 72 : IF (right != end_right) {
3227 [ + - ]: 36 : val_right = *right;
3228 : 108 : } ELSE {
3229 : 36 : val_right = '\0';
3230 : 36 : };
3231 : :
3232 : : /* Compare by current character. */
3233 [ + - + - : 36 : result = (val_left > val_right).to<int32_t>() - (val_left < val_right).to<int32_t>();
+ - + - +
- + - ]
3234 [ + - + - ]: 36 : BREAK(result != 0); // found first position where strings differ
3235 [ + - + - ]: 36 : BREAK(val_left == 0); // reached end of identical strings
3236 : :
3237 : : /* Advance to next character. */
3238 [ + - ]: 36 : left += reverse ? -1 : 1;
3239 [ + - ]: 36 : right += reverse ? -1 : 1;
3240 [ + - ]: 36 : CONTINUE();
3241 : 36 : }
3242 : :
3243 [ + - ]: 36 : RETURN(result);
3244 : 36 : }
3245 : 36 : d.strncmp_no_terminating_nul = std::move(strncmp_no_terminating_nul);
3246 : 36 : }
3247 : :
3248 : : /*----- Call strncmp_no_terminating_nul function. ------*/
3249 : 36 : M_insist(bool(d.strncmp_no_terminating_nul));
3250 : 36 : return (*d.strncmp_no_terminating_nul)(_left.length(), _right.length(), left, right, len);
3251 : : }
3252 : : }
3253 : 54 : };
3254 : :
3255 [ + - + - : 54 : const Var<Ptr<Charx1>> left(_left.val()), right(_right.val());
+ - ]
3256 [ + - + - : 54 : if (_left.can_be_null() or _right.can_be_null()) {
+ - + - ]
3257 [ # # ]: 0 : _Var<I32x1> result; // always set here
3258 [ # # # # : 0 : IF (left.is_null() or right.is_null()) {
# # # # ]
3259 [ # # ]: 0 : result = _I32x1::Null();
3260 [ # # ]: 0 : } ELSE {
3261 [ # # # # : 0 : result = strncmp_non_null(left, right, len);
# # # # ]
3262 : 0 : };
3263 [ # # ]: 0 : return result;
3264 : 0 : } else {
3265 [ + - + - : 54 : const Var<I32x1> result(strncmp_non_null(left, right, len)); // to prevent duplicated computation due to `clone()`
+ - + - +
- ]
3266 [ + - - + ]: 54 : return _I32x1(result);
3267 : 54 : }
3268 : 54 : }
3269 : :
3270 : 12 : _I32x1 m::wasm::strcmp(NChar left, NChar right, bool reverse)
3271 : : {
3272 : : /* Delegate to `strncmp` with length set to minimum of both string lengths **plus** 1 since we need to check if
3273 : : * one string is a prefix of the other, i.e. all of its characters are equal but it is shorter than the other. */
3274 : 12 : U32x1 len(std::min<uint32_t>(left.length(), right.length()) + 1U);
3275 [ + - + - : 12 : return strncmp(left, right, len, reverse);
+ - - + ]
3276 : 12 : }
3277 : :
3278 : 0 : _Boolx1 m::wasm::strncmp(NChar left, NChar right, U32x1 len, cmp_op op, bool reverse)
3279 : : {
3280 [ # # # # : 0 : _I32x1 res = strncmp(left, right, len, reverse);
# # ]
3281 : :
3282 [ # # # # : 0 : switch (op) {
# # # ]
3283 [ # # ]: 0 : case EQ: return res == 0;
3284 [ # # ]: 0 : case NE: return res != 0;
3285 [ # # ]: 0 : case LT: return res < 0;
3286 [ # # ]: 0 : case LE: return res <= 0;
3287 [ # # ]: 0 : case GT: return res > 0;
3288 [ # # ]: 0 : case GE: return res >= 0;
3289 : 0 : }
3290 : 0 : }
3291 : :
3292 : 0 : _Boolx1 m::wasm::strcmp(NChar left, NChar right, cmp_op op, bool reverse)
3293 : : {
3294 [ # # # # ]: 0 : _I32x1 res = strcmp(left, right, reverse);
3295 : :
3296 [ # # # # : 0 : switch (op) {
# # # ]
3297 [ # # ]: 0 : case EQ: return res == 0;
3298 [ # # ]: 0 : case NE: return res != 0;
3299 [ # # ]: 0 : case LT: return res < 0;
3300 [ # # ]: 0 : case LE: return res <= 0;
3301 [ # # ]: 0 : case GT: return res > 0;
3302 [ # # ]: 0 : case GE: return res >= 0;
3303 : 0 : }
3304 : 0 : }
3305 : :
3306 : :
3307 : : /*======================================================================================================================
3308 : : * string copy
3309 : : *====================================================================================================================*/
3310 : :
3311 : 8 : Ptr<Charx1> m::wasm::strncpy(Ptr<Charx1> dst, Ptr<Charx1> src, U32x1 count)
3312 : : {
3313 : : static thread_local struct {} _; // unique caller handle
3314 : : struct data_t : GarbageCollectedData
3315 : : {
3316 : : public:
3317 : : std::optional<FunctionProxy<char*(char*, char*, uint32_t)>> strncpy;
3318 : :
3319 : 8 : data_t(GarbageCollectedData &&d) : GarbageCollectedData(std::move(d)) { }
3320 : : };
3321 : 8 : auto &d = Module::Get().add_garbage_collected_data<data_t>(&_); // garbage collect the `data_t` instance
3322 : :
3323 [ - + ]: 8 : if (not d.strncpy) {
3324 : : /*----- Create function to compute the result. -----*/
3325 [ + - + - : 16 : FUNCTION(strncpy, char*(char*, char*, uint32_t))
+ - ]
3326 : : {
3327 [ + - + - ]: 8 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
3328 : :
3329 [ + - ]: 8 : auto dst = PARAMETER(0);
3330 [ + - ]: 8 : auto src = PARAMETER(1);
3331 [ + - ]: 8 : const auto count = PARAMETER(2);
3332 : :
3333 [ + - + - : 8 : Wasm_insist(not src.is_nullptr(), "source must not be nullptr");
+ - + - ]
3334 [ + - + - : 8 : Wasm_insist(not dst.is_nullptr(), "destination must not be nullptr");
+ - + - ]
3335 : :
3336 [ + - + - : 8 : Var<Ptr<Charx1>> src_end(src + count.make_signed());
+ - ]
3337 [ + - + - : 16 : WHILE (src != src_end) {
+ - + - ]
3338 [ + - + - : 8 : *dst = *src;
+ - ]
3339 [ + - + - : 8 : BREAK(*src == '\0'); // break on terminating NUL byte
+ - ]
3340 [ + - ]: 8 : src += 1;
3341 [ + - ]: 8 : dst += 1;
3342 : : }
3343 : :
3344 [ + - ]: 8 : RETURN(dst);
3345 : 8 : }
3346 : 8 : d.strncpy = std::move(strncpy);
3347 : 8 : }
3348 : :
3349 : : /*----- Call strncpy function. ------*/
3350 : 8 : M_insist(bool(d.strncpy));
3351 [ + - + - : 8 : const Var<Ptr<Charx1>> result((*d.strncpy)(dst, src, count)); // to prevent duplicated computation due to `clone()`
+ - + - ]
3352 [ + - ]: 8 : return result;
3353 : 8 : }
3354 : :
3355 : :
3356 : : /*======================================================================================================================
3357 : : * WasmLike
3358 : : *====================================================================================================================*/
3359 : :
3360 : 0 : _Boolx1 m::wasm::like(NChar _str, NChar _pattern, const char escape_char)
3361 : : {
3362 : : static thread_local struct {} _; // unique caller handle
3363 : : struct data_t : GarbageCollectedData
3364 : : {
3365 : : public:
3366 : : std::optional<FunctionProxy<bool(int32_t, int32_t, char*, char*, char)>> like;
3367 : :
3368 : 0 : data_t(GarbageCollectedData &&d) : GarbageCollectedData(std::move(d)) { }
3369 : : };
3370 : 0 : auto &d = Module::Get().add_garbage_collected_data<data_t>(&_); // garbage collect the `data_t` instance
3371 : :
3372 [ # # ]: 0 : M_insist('_' != escape_char and '%' != escape_char, "illegal escape character");
3373 : :
3374 [ # # # # ]: 0 : if (_str.length() == 0 and _pattern.length() == 0) {
3375 : 0 : _str.discard();
3376 : 0 : _pattern.discard();
3377 : 0 : return _Boolx1(true);
3378 : : }
3379 : :
3380 : 0 : auto like_non_null = [&d, &_str, &_pattern, &escape_char](Ptr<Charx1> str, Ptr<Charx1> pattern) -> Boolx1 {
3381 [ # # # # ]: 0 : Wasm_insist(str.clone().not_null(), "string operand must not be NULL");
3382 [ # # # # ]: 0 : Wasm_insist(pattern.clone().not_null(), "pattern operand must not be NULL");
3383 : :
3384 [ # # ]: 0 : if (not d.like) {
3385 : : /*----- Create function to compute the result. -----*/
3386 [ # # # # : 0 : FUNCTION(like, bool(int32_t, int32_t, char*, char*, char))
# # ]
3387 : : {
3388 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
3389 : :
3390 [ # # ]: 0 : const auto len_ty_str = PARAMETER(0);
3391 [ # # ]: 0 : const auto len_ty_pattern = PARAMETER(1);
3392 [ # # ]: 0 : const auto val_str = PARAMETER(2);
3393 [ # # ]: 0 : const auto val_pattern = PARAMETER(3);
3394 [ # # ]: 0 : const auto escape_char = PARAMETER(4);
3395 : :
3396 : : /*----- Allocate memory for the dynamic programming table. -----*/
3397 : : /* Invariant: dp[i][j] == true iff val_pattern[:i] contains val_str[:j]. Row i and column j is located
3398 : : * at dp + (i - 1) * (`length_str` + 1) + (j - 1). */
3399 [ # # # # : 0 : I32x1 num_entries = (len_ty_str + 1) * (len_ty_pattern + 1);
# # ]
3400 [ # # # # : 0 : const Var<Ptr<Boolx1>> dp = Module::Allocator().malloc<bool>(num_entries.clone().make_unsigned());
# # # # ]
3401 : :
3402 : : /*----- Initialize table with all entries set to false. -----*/
3403 [ # # # # ]: 0 : Var<Ptr<Boolx1>> entry(dp.val());
3404 [ # # # # : 0 : WHILE (entry < dp + num_entries.clone()) {
# # # # #
# # # ]
3405 [ # # # # ]: 0 : *entry = false;
3406 [ # # ]: 0 : entry += 1;
3407 : : }
3408 : :
3409 : : /*----- Reset entry pointer to first entry. -----*/
3410 [ # # # # ]: 0 : entry = dp.val();
3411 : :
3412 : : /*----- Create pointers to track locations of current characters of `val_str` and `val_pattern`. -----*/
3413 [ # # ]: 0 : Var<Ptr<Charx1>> str(val_str);
3414 [ # # ]: 0 : Var<Ptr<Charx1>> pattern(val_pattern);
3415 : :
3416 : : /*----- Compute ends of str and pattern. -----*/
3417 : : /* Create constant local variables to ensure correct pointers since `src` and `pattern` will change. */
3418 [ # # # # ]: 0 : const Var<Ptr<Charx1>> end_str(str + len_ty_str);
3419 [ # # # # ]: 0 : const Var<Ptr<Charx1>> end_pattern(pattern + len_ty_pattern);
3420 : :
3421 : : /*----- Create variables for the current byte of str and pattern. -----*/
3422 [ # # # # ]: 0 : Var<Charx1> byte_str, byte_pattern; // always loaded before first access
3423 : :
3424 : : /*----- Initialize first column. -----*/
3425 : : /* Iterate until current byte of pattern is not a `%`-wildcard and set the respective entries to true. */
3426 [ # # # # : 0 : DO_WHILE (byte_pattern == '%') {
# # ]
3427 [ # # # # : 0 : byte_pattern = Select(pattern < end_pattern, *pattern, '\0');
# # # # ]
3428 [ # # # # ]: 0 : *entry = true;
3429 [ # # # # ]: 0 : entry += len_ty_str + 1;
3430 [ # # ]: 0 : pattern += 1;
3431 : : }
3432 : :
3433 : : /*----- Compute entire table. -----*/
3434 : : /* Create variable for the actual length of str. */
3435 [ # # ]: 0 : Var<I32x1> len_str(0);
3436 : :
3437 : : /* Create flag whether the current byte of pattern is not escaped. */
3438 [ # # ]: 0 : Var<Boolx1> is_not_escaped(true);
3439 : :
3440 : : /* Reset entry pointer to second row and second column. */
3441 [ # # # # : 0 : entry = dp + len_ty_str + 2;
# # ]
3442 : :
3443 : : /* Reset pattern to first character. */
3444 [ # # ]: 0 : pattern = val_pattern;
3445 : :
3446 : : /* Load first byte from pattern if in bounds. */
3447 [ # # # # : 0 : byte_pattern = Select(pattern < end_pattern, *pattern, '\0');
# # # # ]
3448 : :
3449 : : /* Create loop iterating as long as the current byte of pattern is not NUL. */
3450 [ # # # # : 0 : WHILE (byte_pattern != '\0') {
# # # # ]
3451 : : /* If current byte of pattern is not escaped and equals `escape_char`, advance pattern to next
3452 : : * byte and load it. Additionally, mark this byte as escaped and check for invalid escape
3453 : : * sequences. */
3454 [ # # # # : 0 : IF (is_not_escaped and byte_pattern == escape_char) {
# # # # ]
3455 : 0 : pattern += 1;
3456 [ # # # # : 0 : byte_pattern = Select(pattern < end_pattern, *pattern, '\0');
# # ]
3457 : :
3458 : : /* Check whether current byte of pattern is a validly escaped character, i.e. `_`, `%` or
3459 : : * `escape_char`. If not, throw an exception. */
3460 [ # # # # : 0 : IF (byte_pattern != '_' and byte_pattern != '%' and byte_pattern != escape_char) {
# # # # #
# ]
3461 : 0 : Throw(exception::invalid_escape_sequence);
3462 : 0 : };
3463 : :
3464 : 0 : is_not_escaped = false;
3465 : 0 : };
3466 : :
3467 : : /* Reset actual length of str. */
3468 [ # # ]: 0 : len_str = 0;
3469 : :
3470 : : /* Load first byte from str if in bounds. */
3471 [ # # # # : 0 : byte_str = Select(str < end_str, *str, '\0');
# # # # ]
3472 : :
3473 : : /* Create loop iterating as long as the current byte of str is not NUL. */
3474 [ # # # # : 0 : WHILE (byte_str != '\0') {
# # # # ]
3475 : : /* Increment actual length of str. */
3476 [ # # ]: 0 : len_str += 1;
3477 : :
3478 [ # # # # : 0 : IF (is_not_escaped and byte_pattern == '%') {
# # ]
3479 : : /* Store disjunction of above and left entry. */
3480 [ # # # # : 0 : *entry = *(entry - (len_ty_str + 1)) or *(entry - 1);
# # # # #
# # # #
# ]
3481 [ # # ]: 0 : } ELSE {
3482 [ # # # # : 0 : IF ((is_not_escaped and byte_pattern == '_') or byte_pattern == byte_str) {
# # # # ]
3483 : : /* Store above left entry. */
3484 [ # # # # : 0 : *entry = *(entry - (len_ty_str + 2));
# # # # ]
3485 : 0 : };
3486 : 0 : };
3487 : :
3488 : : /* Advance entry pointer to next entry, advance str to next byte, and load next byte from str
3489 : : * if in bounds. */
3490 [ # # ]: 0 : entry += 1;
3491 [ # # ]: 0 : str += 1;
3492 [ # # # # : 0 : byte_str = Select(str < end_str, *str, '\0');
# # # # ]
3493 : : }
3494 : :
3495 : : /* Advance entry pointer to second column in the next row, reset str to first character, advance
3496 : : * pattern to next byte, load next byte from pattern if in bounds, and reset is_not_escaped to
3497 : : * true. */
3498 [ # # # # : 0 : entry += len_ty_str + 1 - len_str;
# # ]
3499 [ # # ]: 0 : str = val_str;
3500 [ # # ]: 0 : pattern += 1;
3501 [ # # # # : 0 : byte_pattern = Select(pattern < end_pattern, *pattern, '\0');
# # # # ]
3502 [ # # ]: 0 : is_not_escaped = true;
3503 : : }
3504 : :
3505 : : /*----- Compute result. -----*/
3506 : : /* Entry pointer points currently to the second column in the first row after the pattern has ended.
3507 : : * Therefore, we have to go one row up and len_str - 1 columns to the right, i.e. the result is
3508 : : * located at entry - (`length_str` + 1) + len_str - 1 = entry + len_str - (`length_str` + 2). */
3509 [ # # # # : 0 : const Var<Boolx1> result(*(entry + len_str - (len_ty_str + 2)));
# # # # #
# ]
3510 : :
3511 : : /*----- Free allocated space. -----*/
3512 [ # # # # : 0 : Module::Allocator().free(dp, num_entries.make_unsigned());
# # ]
3513 : :
3514 [ # # ]: 0 : RETURN(result);
3515 : 0 : }
3516 : :
3517 : 0 : d.like = std::move(like);
3518 : 0 : }
3519 : :
3520 : : /*----- Call like function. ------*/
3521 : 0 : M_insist(bool(d.like));
3522 : 0 : return (*d.like)(_str.length(), _pattern.length(), str, pattern, escape_char);
3523 : 0 : };
3524 : :
3525 [ # # # # ]: 0 : if (_str.can_be_null() or _pattern.can_be_null()) {
3526 : 0 : auto [_val_str, is_null_str] = _str.split();
3527 [ # # ]: 0 : auto [_val_pattern, is_null_pattern] = _pattern.split();
3528 [ # # # # ]: 0 : Ptr<Charx1> val_str(_val_str), val_pattern(_val_pattern); // since structured bindings cannot be used in lambda capture
3529 : :
3530 [ # # ]: 0 : _Var<Boolx1> result; // always set here
3531 [ # # # # : 0 : IF (is_null_str or is_null_pattern) {
# # ]
3532 [ # # ]: 0 : result = _Boolx1::Null();
3533 [ # # ]: 0 : } ELSE {
3534 [ # # # # : 0 : result = like_non_null(val_str, val_pattern);
# # ]
3535 : 0 : };
3536 [ # # ]: 0 : return result;
3537 : 0 : } else {
3538 [ # # # # : 0 : const Var<Boolx1> result(like_non_null(_str, _pattern)); // to prevent duplicated computation due to `clone()`
# # ]
3539 [ # # # # ]: 0 : return _Boolx1(result);
3540 : 0 : }
3541 : 0 : }
3542 : :
3543 : 0 : _Boolx1 m::wasm::like_contains(NChar _str, const ThreadSafePooledString &_pattern)
3544 : : {
3545 : : static thread_local struct {} _; // unique caller handle
3546 : : struct data_t : GarbageCollectedData
3547 : : {
3548 : : public:
3549 : : ///> one function per static pattern
3550 : : std::unordered_map<ThreadSafePooledString, FunctionProxy<bool(int32_t, char*)>> contains_map;
3551 : :
3552 : 0 : data_t(GarbageCollectedData &&d) : GarbageCollectedData(std::move(d)) { }
3553 : : };
3554 : 0 : auto &d = Module::Get().add_garbage_collected_data<data_t>(&_); // garbage collect the `data_t` instance
3555 : :
3556 [ # # # # ]: 0 : M_insist(std::regex_match(*_pattern, std::regex("%[^_%\\\\]+%")), "invalid contains pattern");
3557 : :
3558 [ # # ]: 0 : if (_str.length() == 0) {
3559 : 0 : _str.discard();
3560 : 0 : return _Boolx1(false);
3561 : : }
3562 : :
3563 : 0 : auto contains_non_null = [&d, &_str, &_pattern](Ptr<Charx1> str) -> Boolx1 {
3564 [ # # # # ]: 0 : Wasm_insist(str.clone().not_null(), "string operand must not be NULL");
3565 : :
3566 : 0 : auto it = d.contains_map.find(_pattern);
3567 [ # # ]: 0 : if (it == d.contains_map.end()) {
3568 : : /*----- Create function to compute the result. -----*/
3569 [ # # # # : 0 : FUNCTION(contains, bool(int32_t, char*))
# # ]
3570 : : {
3571 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
3572 : :
3573 [ # # ]: 0 : const auto len_ty_str = PARAMETER(0);
3574 [ # # ]: 0 : auto val_str = PARAMETER(1);
3575 : :
3576 : : /*----- Copy pattern without enclosing `%` to make it accessible with runtime offset. -----*/
3577 [ # # ]: 0 : const int32_t len_pattern = strlen(*_pattern) - 2; // minus 2 due to enclosing `%`
3578 [ # # # # ]: 0 : auto pattern = Module::Allocator().raw_malloc<char>(len_pattern);
3579 [ # # ]: 0 : for (std::size_t i = 0; i < len_pattern; ++i)
3580 [ # # ]: 0 : pattern[i] = (*_pattern)[i + 1]; // access _pattern with offset +1 due to starting `%`
3581 : :
3582 : : /*----- Precompute prefix table. -----*/
3583 [ # # # # ]: 0 : auto tbl = Module::Allocator().raw_malloc<int32_t>(len_pattern + 1);
3584 : 0 : int32_t len_prefix = -1;
3585 : :
3586 : 0 : tbl[0] = len_prefix;
3587 [ # # ]: 0 : for (std::size_t i = 1; i < len_pattern + 1; ++i) {
3588 [ # # # # ]: 0 : while (len_prefix >= 0 and pattern[len_prefix] != pattern[i - 1])
3589 : 0 : len_prefix = tbl[len_prefix];
3590 : 0 : ++len_prefix;
3591 : 0 : tbl[i] = len_prefix;
3592 : 0 : }
3593 : :
3594 : : /*----- Search pattern in string. -----*/
3595 [ # # # # ]: 0 : const Var<Ptr<Charx1>> end_str(val_str + len_ty_str);
3596 [ # # ]: 0 : Var<I32x1> pos_pattern(0);
3597 [ # # # # : 0 : WHILE (val_str < end_str and *val_str != '\0') {
# # # # #
# # # #
# ]
3598 [ # # # # : 0 : WHILE(pos_pattern >= 0 and *val_str != *(Ptr<Charx1>(pattern) + pos_pattern)) {
# # # # #
# # # # #
# # # # #
# ]
3599 [ # # # # : 0 : Wasm_insist(pos_pattern < len_pattern + 1);
# # ]
3600 [ # # # # : 0 : pos_pattern = *(Ptr<I32x1>(tbl) + pos_pattern);
# # # # ]
3601 : : }
3602 [ # # ]: 0 : val_str += 1;
3603 [ # # ]: 0 : pos_pattern += 1;
3604 [ # # # # ]: 0 : IF (pos_pattern == len_pattern) {
3605 : 0 : RETURN(true);
3606 : 0 : };
3607 : : }
3608 [ # # ]: 0 : RETURN(false);
3609 : 0 : }
3610 [ # # ]: 0 : it = d.contains_map.emplace_hint(it, _pattern, std::move(contains));
3611 : 0 : }
3612 : :
3613 : : /*----- Call contains function. ------*/
3614 : 0 : M_insist(it != d.contains_map.end());
3615 : 0 : return (it->second)(_str.length(), str);
3616 : 0 : };
3617 : :
3618 [ # # ]: 0 : if (_str.can_be_null()) {
3619 : 0 : auto [_val_str, is_null_str] = _str.split();
3620 [ # # ]: 0 : Ptr<Charx1> val_str(_val_str); // since structured bindings cannot be used in lambda capture
3621 : :
3622 [ # # ]: 0 : _Var<Boolx1> result; // always set here
3623 [ # # ]: 0 : IF (is_null_str) {
3624 [ # # ]: 0 : result = _Boolx1::Null();
3625 [ # # ]: 0 : } ELSE {
3626 [ # # # # ]: 0 : result = contains_non_null(val_str);
3627 : 0 : };
3628 [ # # ]: 0 : return result;
3629 : 0 : } else {
3630 [ # # # # ]: 0 : const Var<Boolx1> result(contains_non_null(_str)); // to prevent duplicated computation due to `clone()`
3631 [ # # # # ]: 0 : return _Boolx1(result);
3632 : 0 : }
3633 : 0 : }
3634 : :
3635 : 0 : _Boolx1 m::wasm::like_prefix(NChar str, const ThreadSafePooledString &pattern)
3636 : : {
3637 [ # # # # ]: 0 : M_insist(std::regex_match(*pattern, std::regex("[^_%\\\\]+%")), "invalid prefix pattern");
3638 : :
3639 : : /*----- Create lower bound. -----*/
3640 : 0 : const int32_t len_pattern = strlen(*pattern) - 1; // minus 1 due to ending `%`
3641 : 0 : auto _lower_bound = Module::Allocator().raw_malloc<char>(len_pattern + 1);
3642 [ # # ]: 0 : for (std::size_t i = 0; i < len_pattern; ++i)
3643 : 0 : _lower_bound[i] = (*pattern)[i];
3644 : 0 : _lower_bound[len_pattern] = '\0';
3645 [ # # ]: 0 : NChar lower_bound(Ptr<Charx1>(_lower_bound), false, len_pattern, true);
3646 : :
3647 : : /*----- Create upper bound. -----*/
3648 [ # # # # ]: 0 : auto _upper_bound = Module::Allocator().raw_malloc<char>(len_pattern + 1);
3649 [ # # ]: 0 : for (std::size_t i = 0; i < len_pattern - 1; ++i)
3650 [ # # ]: 0 : _upper_bound[i] = (*pattern)[i];
3651 [ # # ]: 0 : const char last_char = (*pattern)[len_pattern - 1];
3652 : 0 : _upper_bound[len_pattern - 1] = last_char + 1; // increment last character for upper bound
3653 : 0 : _upper_bound[len_pattern] = '\0';
3654 [ # # # # ]: 0 : NChar upper_bound(Ptr<Charx1>(_upper_bound), false, len_pattern, true);
3655 : :
3656 : : /*----- Compute result by checking whether given string is in created interval. -----*/
3657 [ # # ]: 0 : auto str_cpy = str.clone();
3658 [ # # # # : 0 : return strcmp(str_cpy, lower_bound, GE) and strcmp(str, upper_bound, LT);
# # # # #
# # # #
# ]
3659 : 0 : }
3660 : :
3661 : 0 : _Boolx1 m::wasm::like_suffix(NChar str, const ThreadSafePooledString &pattern)
3662 : : {
3663 [ # # # # ]: 0 : M_insist(std::regex_match(*pattern, std::regex("%[^_%\\\\]+")), "invalid suffix pattern");
3664 : :
3665 : : /*----- Create lower bound. -----*/
3666 : 0 : const int32_t len_pattern = strlen(*pattern) - 1; // minus 1 due to starting `%`
3667 : 0 : auto _lower_bound = Module::Allocator().raw_malloc<char>(len_pattern + 1);
3668 [ # # ]: 0 : for (std::size_t i = 0; i < len_pattern; ++i)
3669 : 0 : _lower_bound[i] = (*pattern)[i + 1]; // access pattern with offset +1 due to starting `%`
3670 : 0 : _lower_bound[len_pattern] = '\0';
3671 [ # # ]: 0 : NChar lower_bound(Ptr<Charx1>(_lower_bound), false, len_pattern, true);
3672 : :
3673 : : /*----- Create upper bound. -----*/
3674 [ # # # # ]: 0 : auto _upper_bound = Module::Allocator().raw_malloc<char>(len_pattern + 1);
3675 [ # # ]: 0 : const char first_char = (*pattern)[1]; // access first character at offset 1 due to starting `%`
3676 : 0 : _upper_bound[0] = first_char + 1; // increment first character for upper bound
3677 [ # # ]: 0 : for (std::size_t i = 1; i < len_pattern; ++i)
3678 [ # # ]: 0 : _upper_bound[i] = (*pattern)[i + 1]; // access pattern with offset +1 due to starting `%`
3679 : 0 : _upper_bound[len_pattern] = '\0';
3680 [ # # # # ]: 0 : NChar upper_bound(Ptr<Charx1>(_upper_bound), false, len_pattern, true);
3681 : :
3682 : : /*----- Compute result by checking whether given string is in created interval when reversed. -----*/
3683 [ # # # # ]: 0 : const auto max_length = std::max<uint32_t>(str.length(), len_pattern); // use maximal length due to reversed strncmp
3684 [ # # ]: 0 : auto str_cpy = str.clone();
3685 [ # # # # : 0 : return strncmp(str_cpy, lower_bound, U32x1(max_length), GE, true) and
# # # # #
# ]
3686 [ # # # # : 0 : strncmp(str, upper_bound, U32x1(max_length), LT, true);
# # # # ]
3687 : 0 : }
3688 : :
3689 : :
3690 : : /*======================================================================================================================
3691 : : * comparator
3692 : : *====================================================================================================================*/
3693 : :
3694 : : template<bool Predicated>
3695 : 0 : I32x1 m::wasm::compare(const Environment &env_left, const Environment &env_right,
3696 : : const std::vector<SortingOperator::order_type> &order)
3697 : : {
3698 : : if constexpr (Predicated) {
3699 : 0 : Var<I32x1> result(0); // explicitly (re-)set result to 0
3700 : :
3701 : : /*----- Compile ordering. -----*/
3702 [ # # ]: 0 : for (auto &o : order) {
3703 : : /*----- Compile order expression for left tuple. -----*/
3704 [ # # ]: 0 : SQL_t _val_left = env_left.template compile(o.first);
3705 : :
3706 [ # # # # ]: 0 : std::visit(overloaded {
3707 : 0 : [&]<typename T>(Expr<T> val_left) -> void {
3708 : : /*----- Compile order expression for right tuple. -----*/
3709 : 0 : Expr<T> val_right = env_right.template compile<Expr<T>>(o.first);
3710 : :
3711 [ # # # # : 0 : M_insist(val_left.can_be_null() == val_right.can_be_null(),
# # # # #
# # # #
# ]
3712 : : "either both or none of the value to compare must be nullable");
3713 [ # # # # : 0 : if (val_left.can_be_null()) {
# # # # #
# # # #
# ]
3714 : : using type = std::conditional_t<std::is_same_v<T, bool>, _I32x1, Expr<T>>;
3715 [ # # # # : 0 : Var<type> left, right;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3716 : : if constexpr (std::is_same_v<T, bool>) {
3717 [ # # # # ]: 0 : left = val_left.template to<int32_t>();
3718 [ # # # # ]: 0 : right = val_right.template to<int32_t>();
3719 : : } else {
3720 [ # # # # : 0 : left = val_left;
# # # # #
# # # ]
3721 [ # # # # : 0 : right = val_right;
# # # # #
# # # ]
3722 : : }
3723 : :
3724 : : /*----- Compare both with current order expression and update result. -----*/
3725 [ # # # # : 0 : I32x1 cmp_null = right.is_null().template to<int32_t>() - left.is_null().template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3726 [ # # # # : 0 : _I32x1 _val_lt = (left < right).template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3727 [ # # # # : 0 : _I32x1 _val_gt = (left > right).template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3728 [ # # # # : 0 : _I32x1 _cmp_val = o.second ? _val_gt - _val_lt : _val_lt - _val_gt;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3729 [ # # # # : 0 : auto [cmp_val, cmp_is_null] = _cmp_val.split();
# # # # #
# # # #
# ]
3730 [ # # # # : 0 : cmp_is_null.discard();
# # # # #
# # # #
# ]
3731 [ # # # # : 0 : I32x1 cmp = (cmp_null << 1) + cmp_val; // potentially-null value of comparison is overruled by cmp_null
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3732 [ # # # # : 0 : result <<= 2; // shift result s.t. first difference will determine order
# # # # #
# # # #
# ]
3733 [ # # # # : 0 : result += cmp; // add current comparison to result
# # # # #
# # # #
# ]
3734 : 0 : } else {
3735 : : using type = std::conditional_t<std::is_same_v<T, bool>, I32x1, PrimitiveExpr<T>>;
3736 [ # # # # : 0 : Var<type> left, right;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3737 : : if constexpr (std::is_same_v<T, bool>) {
3738 [ # # # # : 0 : left = val_left.insist_not_null().template to<int32_t>();
# # ]
3739 [ # # # # : 0 : right = val_right.insist_not_null().template to<int32_t>();
# # ]
3740 : : } else {
3741 [ # # # # : 0 : left = val_left.insist_not_null();
# # # # #
# # # # #
# # # # #
# # # #
# ]
3742 [ # # # # : 0 : right = val_right.insist_not_null();
# # # # #
# # # # #
# # # # #
# # # #
# ]
3743 : : }
3744 : :
3745 : : /*----- Compare both with current order expression and update result. -----*/
3746 [ # # # # : 0 : I32x1 val_lt = (left < right).template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3747 [ # # # # : 0 : I32x1 val_gt = (left > right).template to<int32_t>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3748 [ # # # # : 0 : I32x1 cmp = o.second ? val_gt - val_lt : val_lt - val_gt;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3749 [ # # # # : 0 : result <<= 1; // shift result s.t. first difference will determine order
# # # # #
# # # #
# ]
3750 [ # # # # : 0 : result += cmp; // add current comparison to result
# # # # #
# # # #
# ]
3751 : 0 : }
3752 : 0 : },
3753 : 0 : [&](NChar val_left) -> void {
3754 : 0 : auto &cs = as<const CharacterSequence>(*o.first.get().type());
3755 : :
3756 : : /*----- Compile order expression for right tuple. -----*/
3757 : 0 : NChar val_right = env_right.template compile<NChar>(o.first);
3758 : :
3759 [ # # # # : 0 : Var<Ptr<Charx1>> _left(val_left.val()), _right(val_right.val());
# # # # ]
3760 [ # # # # ]: 0 : NChar left(_left, val_left.can_be_null(), val_left.length(), val_left.guarantees_terminating_nul()),
3761 [ # # # # ]: 0 : right(_right, val_right.can_be_null(), val_right.length(), val_right.guarantees_terminating_nul());
3762 : :
3763 [ # # ]: 0 : M_insist(val_left.can_be_null() == val_right.can_be_null(),
3764 : : "either both or none of the value to compare must be nullable");
3765 [ # # ]: 0 : if (val_left.can_be_null()) {
3766 : : /*----- Compare both with current order expression and update result. -----*/
3767 [ # # # # : 0 : I32x1 cmp_null = _right.is_null().to<int32_t>() - _left.is_null().to<int32_t>();
# # # # #
# ]
3768 [ # # # # : 0 : _I32x1 _delta = o.second ? strcmp(left, right) : strcmp(right, left);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3769 [ # # ]: 0 : auto [delta_val, delta_is_null] = _delta.split();
3770 [ # # # # : 0 : Wasm_insist(delta_val.clone() >= -1 and delta_val.clone() <= 1,
# # # # #
# # # #
# ]
3771 : : "result of strcmp is assumed to be in [-1,1]");
3772 [ # # ]: 0 : delta_is_null.discard();
3773 [ # # # # : 0 : I32x1 cmp = (cmp_null << 1) + delta_val; // potentially-null value of comparison is overruled by cmp_null
# # ]
3774 [ # # ]: 0 : result <<= 2; // shift result s.t. first difference will determine order
3775 [ # # ]: 0 : result += cmp; // add current comparison to result
3776 : 0 : } else {
3777 : : /*----- Compare both with current order expression and update result. -----*/
3778 [ # # # # : 0 : I32x1 delta = o.second ? strcmp(left, right).insist_not_null()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3779 [ # # # # : 0 : : strcmp(right, left).insist_not_null();
# # # # ]
3780 [ # # # # : 0 : Wasm_insist(delta.clone() >= -1 and delta.clone() <= 1,
# # # # #
# # # #
# ]
3781 : : "result of strcmp is assumed to be in [-1,1]");
3782 [ # # ]: 0 : result <<= 1; // shift result s.t. first difference will determine order
3783 [ # # ]: 0 : result += delta; // add current comparison to result
3784 : 0 : }
3785 : 0 : },
3786 : 0 : [](auto&&) -> void { M_unreachable("SIMDfication currently not supported"); },
3787 : 0 : [](std::monostate) -> void { M_unreachable("invalid expression"); }
3788 : : }, _val_left);
3789 : 0 : }
3790 : :
3791 [ # # ]: 0 : return result;
3792 : 0 : } else {
3793 : 0 : Var<I32x1> result; // always set here
3794 : :
3795 : : /*----- Compile ordering. -----*/
3796 [ # # # # ]: 0 : BLOCK(compare) {
3797 : 0 : auto emit_comparison_rec = [&](decltype(order.cbegin()) curr, const decltype(order.cend()) end,
3798 : : auto &rec) -> void
3799 : : {
3800 : : /*----- If end of ordering is reached, left and right tuple are equal. -----*/
3801 [ # # ]: 0 : if (curr == end) {
3802 : 0 : result = 0;
3803 : 0 : return;
3804 : : }
3805 : :
3806 : : /*----- Compile order expression for left tuple. -----*/
3807 : 0 : SQL_t _val_left = env_left.template compile(curr->first);
3808 : :
3809 [ # # # # ]: 0 : std::visit(overloaded {
3810 : 0 : [&]<typename T>(Expr<T> val_left) -> void {
3811 : : /*----- Compile order expression for right tuple. -----*/
3812 : 0 : Expr<T> val_right = env_right.template compile<Expr<T>>(curr->first);
3813 : :
3814 [ # # # # : 0 : M_insist(val_left.can_be_null() == val_right.can_be_null(),
# # # # #
# # # #
# ]
3815 : : "either both or none of the value to compare must be nullable");
3816 [ # # # # : 0 : if (val_left.can_be_null()) {
# # # # #
# # # #
# ]
3817 : : using type = std::conditional_t<std::is_same_v<T, bool>, _I32x1, Expr<T>>;
3818 [ # # # # : 0 : Var<type> _left, _right;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3819 : : if constexpr (std::is_same_v<T, bool>) {
3820 [ # # # # ]: 0 : _left = val_left.template to<int32_t>();
3821 [ # # # # ]: 0 : _right = val_right.template to<int32_t>();
3822 : : } else {
3823 [ # # # # : 0 : _left = val_left;
# # # # #
# # # ]
3824 [ # # # # : 0 : _right = val_right;
# # # # #
# # # ]
3825 : : }
3826 : :
3827 : : /*----- Compare both with current order expression and potentially set result. -----*/
3828 [ # # # # : 0 : IF (_left.not_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3829 [ # # # # : 0 : IF (_right.is_null()) {
# # # # #
# # # #
# ]
3830 : 0 : result = 1;
3831 : 0 : GOTO(compare);
3832 : 0 : };
3833 [ # # # # : 0 : auto left = _left.val().insist_not_null(),
# # # # #
# # # #
# ]
3834 [ # # # # : 0 : right = _right.val().insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3835 [ # # # # : 0 : Boolx1 left_lt_right = curr->second ? left.clone() < right.clone()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3836 [ # # # # : 0 : : left.clone() > right.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3837 [ # # # # : 0 : IF (left_lt_right) {
# # # # #
# # # #
# ]
3838 : 0 : result = -1;
3839 : 0 : GOTO(compare);
3840 : 0 : };
3841 [ # # # # : 0 : Boolx1 left_gt_right = curr->second ? left > right : left < right;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3842 [ # # # # : 0 : IF (left_gt_right) {
# # # # #
# # # #
# ]
3843 : 0 : result = 1;
3844 : 0 : GOTO(compare);
3845 : 0 : };
3846 [ # # # # : 0 : } ELSE {
# # # # #
# # # #
# ]
3847 [ # # # # : 0 : IF (_right.not_null()) {
# # # # #
# # # #
# ]
3848 : 0 : result = -1;
3849 : 0 : GOTO(compare);
3850 : 0 : };
3851 : 0 : };
3852 : 0 : } else {
3853 : : using type = std::conditional_t<std::is_same_v<T, bool>, I32x1, PrimitiveExpr<T>>;
3854 [ # # # # : 0 : Var<type> left, right;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3855 : : if constexpr (std::is_same_v<T, bool>) {
3856 [ # # # # : 0 : left = val_left.insist_not_null().template to<int32_t>();
# # ]
3857 [ # # # # : 0 : right = val_right.insist_not_null().template to<int32_t>();
# # ]
3858 : : } else {
3859 [ # # # # : 0 : left = val_left.insist_not_null();
# # # # #
# # # # #
# # # # #
# # # #
# ]
3860 [ # # # # : 0 : right = val_right.insist_not_null();
# # # # #
# # # # #
# # # # #
# # # #
# ]
3861 : : }
3862 : :
3863 : : /*----- Compare both with current order expression and potentially set result. -----*/
3864 [ # # # # : 0 : Boolx1 left_lt_right = curr->second ? left < right : left > right;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3865 [ # # # # : 0 : IF (left_lt_right) {
# # # # #
# # # #
# ]
3866 : 0 : result = -1;
3867 : 0 : GOTO(compare);
3868 : 0 : };
3869 [ # # # # : 0 : Boolx1 left_gt_right = curr->second ? left > right : left < right;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3870 [ # # # # : 0 : IF (left_gt_right) {
# # # # #
# # # #
# ]
3871 : 0 : result = 1;
3872 : 0 : GOTO(compare);
3873 : 0 : };
3874 : 0 : }
3875 : 0 : },
3876 : 0 : [&](NChar val_left) -> void {
3877 : 0 : auto &cs = as<const CharacterSequence>(*curr->first.get().type());
3878 : :
3879 : : /*----- Compile order expression for right tuple. -----*/
3880 : 0 : NChar val_right = env_right.template compile<NChar>(curr->first);
3881 : :
3882 [ # # # # : 0 : Var<Ptr<Charx1>> _left(val_left.val()), _right(val_right.val());
# # # # ]
3883 : : ///> create non-nullable `NChar`s since NULL checks are done explicitly here before using them
3884 [ # # # # ]: 0 : NChar left(_left, false, val_left.length(), val_left.guarantees_terminating_nul()),
3885 [ # # # # ]: 0 : right(_right, false, val_right.length(), val_right.guarantees_terminating_nul());
3886 : :
3887 [ # # ]: 0 : M_insist(val_left.can_be_null() == val_right.can_be_null(),
3888 : : "either both or none of the value to compare must be nullable");
3889 [ # # ]: 0 : if (val_left.can_be_null()) {
3890 : : /*----- Compare both with current order expression and potentially set result. -----*/
3891 [ # # # # : 0 : IF (_left.not_null()) {
# # ]
3892 [ # # ]: 0 : IF (_right.is_null()) {
3893 : 0 : result = 1;
3894 : 0 : GOTO(compare);
3895 : 0 : };
3896 [ # # # # : 0 : I32x1 cmp = curr->second ? strcmp(left, right).insist_not_null()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3897 [ # # # # : 0 : : strcmp(right, left).insist_not_null();
# # # # ]
3898 [ # # # # : 0 : IF (cmp.clone() != 0) {
# # # # ]
3899 : 0 : result = cmp;
3900 : 0 : GOTO(compare);
3901 : 0 : };
3902 [ # # ]: 0 : } ELSE {
3903 [ # # ]: 0 : IF (_right.not_null()) {
3904 : 0 : result = -1;
3905 : 0 : GOTO(compare);
3906 : 0 : };
3907 : 0 : };
3908 : 0 : } else {
3909 : : /*----- Compare both with current order expression and potentially set result. -----*/
3910 [ # # # # : 0 : I32x1 cmp = curr->second ? strcmp(left, right).insist_not_null()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3911 [ # # # # : 0 : : strcmp(right, left).insist_not_null();
# # # # ]
3912 [ # # # # : 0 : IF (cmp.clone() != 0) {
# # # # ]
3913 : 0 : result = cmp;
3914 : 0 : GOTO(compare);
3915 : 0 : };
3916 : 0 : }
3917 : 0 : },
3918 : 0 : [](auto&&) -> void { M_unreachable("SIMDfication currently not supported"); },
3919 : 0 : [](std::monostate) -> void { M_unreachable("invalid expression"); }
3920 : : }, _val_left);
3921 : :
3922 : : /*----- Recurse to next comparison. -----*/
3923 [ # # # # ]: 0 : rec(std::next(curr), end, rec);
3924 : 0 : };
3925 [ # # ]: 0 : emit_comparison_rec(order.cbegin(), order.cend(), emit_comparison_rec);
3926 : : }
3927 : :
3928 : : /* GOTOs from above jump here */
3929 : :
3930 [ # # ]: 0 : return result;
3931 : 0 : }
3932 : 0 : }
3933 : :
3934 : : // explicit instantiations to prevent linker errors
3935 : : template I32x1 m::wasm::compare<false>(
3936 : : const Environment&, const Environment&, const std::vector<SortingOperator::order_type>&
3937 : : );
3938 : : template I32x1 m::wasm::compare<true>(
3939 : : const Environment&, const Environment&, const std::vector<SortingOperator::order_type>&
3940 : : );
|
