Branch data Line data Source code
1 : : #include "backend/WasmOperator.hpp"
2 : :
3 : : #include "backend/Interpreter.hpp"
4 : : #include "backend/WasmAlgo.hpp"
5 : : #include "backend/WasmMacro.hpp"
6 : : #include <mutable/catalog/Catalog.hpp>
7 : : #include <mutable/parse/AST.hpp>
8 : : #include <mutable/util/fn.hpp>
9 : : #include <numeric>
10 : :
11 : :
12 : : using namespace m;
13 : : using namespace m::ast;
14 : : using namespace m::storage;
15 : : using namespace m::wasm;
16 : :
17 : :
18 : : /*======================================================================================================================
19 : : * CLI arguments
20 : : *====================================================================================================================*/
21 : :
22 : : namespace {
23 : :
24 : : __attribute__((constructor(201)))
25 : 1 : static void add_wasm_operator_args()
26 : : {
27 : 1 : Catalog &C = Catalog::Get();
28 : :
29 : : /*----- Command-line arguments -----*/
30 : 1 : C.arg_parser().add<std::vector<std::string_view>>(
31 : : /* group= */ "Wasm",
32 : : /* short= */ nullptr,
33 : : /* long= */ "--scan-implementations",
34 : : /* description= */ "a comma seperated list of physical scan implementations to consider (`Scan` or `IndexScan`)",
35 : 0 : /* callback= */ [](std::vector<std::string_view> impls){
36 : 0 : options::scan_implementations = option_configs::ScanImplementation(0UL);
37 [ # # ]: 0 : for (const auto &elem : impls) {
38 [ # # ]: 0 : if (strneq(elem.data(), "Scan", elem.size()))
39 : 0 : options::scan_implementations |= option_configs::ScanImplementation::SCAN;
40 [ # # ]: 0 : else if (strneq(elem.data(), "IndexScan", elem.size()))
41 : 0 : options::scan_implementations |= option_configs::ScanImplementation::INDEX_SCAN;
42 : : else
43 : 0 : std::cerr << "warning: ignore invalid physical scan implementation " << elem << std::endl;
44 : : }
45 : 0 : }
46 : : );
47 : 1 : C.arg_parser().add<std::vector<std::string_view>>(
48 : : /* group= */ "Wasm",
49 : : /* short= */ nullptr,
50 : : /* long= */ "--grouping-implementations",
51 : : /* description= */ "a comma seperated list of physical grouping implementations to consider (`HashBased` or "
52 : : "`Ordered`)",
53 : 0 : /* callback= */ [](std::vector<std::string_view> impls){
54 : 0 : options::grouping_implementations = option_configs::GroupingImplementation(0UL);
55 [ # # ]: 0 : for (const auto &elem : impls) {
56 [ # # ]: 0 : if (strneq(elem.data(), "HashBased", elem.size()))
57 : 0 : options::grouping_implementations |= option_configs::GroupingImplementation::HASH_BASED;
58 [ # # ]: 0 : else if (strneq(elem.data(), "Ordered", elem.size()))
59 : 0 : options::grouping_implementations |= option_configs::GroupingImplementation::ORDERED;
60 : : else
61 : 0 : std::cerr << "warning: ignore invalid physical grouping implementation " << elem << std::endl;
62 : : }
63 : 0 : }
64 : : );
65 : 1 : C.arg_parser().add<std::vector<std::string_view>>(
66 : : /* group= */ "Wasm",
67 : 0 : /* short= */ nullptr,
68 : : /* long= */ "--sorting-implementations",
69 : : /* description= */ "a comma seperated list of physical sorting implementations to consider (`Quicksort` or "
70 : : "`NoOp`)",
71 : 0 : /* callback= */ [](std::vector<std::string_view> impls){
72 : 0 : options::sorting_implementations = option_configs::SortingImplementation(0UL);
73 [ # # ]: 0 : for (const auto &elem : impls) {
74 [ # # ]: 1 : if (strneq(elem.data(), "Quicksort", elem.size()))
75 : 0 : options::sorting_implementations |= option_configs::SortingImplementation::QUICKSORT;
76 [ # # ]: 0 : else if (strneq(elem.data(), "NoOp", elem.size()))
77 : 0 : options::sorting_implementations |= option_configs::SortingImplementation::NOOP;
78 : : else
79 : 0 : std::cerr << "warning: ignore invalid physical sorting implementation " << elem << std::endl;
80 : : }
81 : 0 : }
82 : : );
83 : 1 : C.arg_parser().add<std::vector<std::string_view>>(
84 : : /* group= */ "Wasm",
85 : : /* short= */ nullptr,
86 : : /* long= */ "--join-implementations",
87 : : /* description= */ "a comma seperated list of physical join implementations to consider (`NestedLoops`, "
88 : : "`SimpleHash`, or `SortMerge`)",
89 : 0 : /* callback= */ [](std::vector<std::string_view> impls){
90 : 0 : options::join_implementations = option_configs::JoinImplementation(0UL);
91 [ # # ]: 0 : for (const auto &elem : impls) {
92 [ # # ]: 0 : if (strneq(elem.data(), "NestedLoops", elem.size()))
93 : 0 : options::join_implementations |= option_configs::JoinImplementation::NESTED_LOOPS;
94 [ # # ]: 0 : else if (strneq(elem.data(), "SimpleHash", elem.size()))
95 : 0 : options::join_implementations |= option_configs::JoinImplementation::SIMPLE_HASH;
96 [ # # ]: 0 : else if (strneq(elem.data(), "SortMerge", elem.size()))
97 : 0 : options::join_implementations |= option_configs::JoinImplementation::SORT_MERGE;
98 : : else
99 : 0 : std::cerr << "warning: ignore invalid physical join implementation " << elem << std::endl;
100 : : }
101 : 0 : }
102 : : );
103 : 1 : C.arg_parser().add<std::vector<std::string_view>>(
104 : : /* group= */ "Wasm",
105 : : /* short= */ nullptr,
106 : : /* long= */ "--index-implementations",
107 : : /* description= */ "a comma separated list of index implementations to consider for index scans (`Array`, or"
108 : : " `Rmi`)",
109 : 0 : /* callback= */ [](std::vector<std::string_view> impls){
110 : 0 : options::index_implementations = option_configs::IndexImplementation(0UL);
111 [ # # ]: 0 : for (const auto &elem : impls) {
112 [ # # ]: 0 : if (strneq(elem.data(), "Array", elem.size()))
113 : 0 : options::index_implementations |= option_configs::IndexImplementation::ARRAY;
114 [ # # ]: 0 : else if (strneq(elem.data(), "Rmi", elem.size()))
115 : 0 : options::index_implementations |= option_configs::IndexImplementation::RMI;
116 : : else
117 : 0 : std::cerr << "warning: ignore invalid index implementation " << elem << std::endl;
118 : : }
119 : 0 : }
120 : : );
121 : 1 : C.arg_parser().add<const char*>(
122 : : /* group= */ "Wasm",
123 : : /* short= */ nullptr,
124 : : /* long= */ "--index-scan-strategy",
125 : : /* description= */ "specify the index scan strategy (`Compilation`, `Interpretation`, or `Hybrid`)",
126 : 0 : /* callback= */ [](const char *strategy){
127 [ # # ]: 0 : if (streq(strategy, "Compilation"))
128 : 0 : options::index_scan_strategy = option_configs::IndexScanStrategy::COMPILATION;
129 [ # # ]: 0 : else if (streq(strategy, "Interpretation"))
130 : 0 : options::index_scan_strategy = option_configs::IndexScanStrategy::INTERPRETATION;
131 [ # # ]: 0 : else if (streq(strategy, "Hybrid"))
132 : 0 : options::index_scan_strategy = option_configs::IndexScanStrategy::HYBRID;
133 : : else
134 : 0 : std::cerr << "warning: ignore invalid index scan strategy " << strategy << std::endl;
135 : 0 : }
136 : : );
137 : 1 : C.arg_parser().add<const char*>(
138 : : /* group= */ "Wasm",
139 : : /* short= */ nullptr,
140 : : /* long= */ "--index-scan-compilation-strategy",
141 : : /* description= */ "specify the materialization strategy for index scans (`Callback` or `ExposedMemory`)",
142 : 0 : /* callback= */ [](const char *strategy){
143 [ # # ]: 0 : if (streq(strategy, "Callback"))
144 : 0 : options::index_scan_compilation_strategy = option_configs::IndexScanCompilationStrategy::CALLBACK;
145 [ # # ]: 0 : else if (streq(strategy, "ExposedMemory"))
146 : 0 : options::index_scan_compilation_strategy = option_configs::IndexScanCompilationStrategy::EXPOSED_MEMORY;
147 : : else
148 : 0 : std::cerr << "warning: ignore invalid index scan strategy " << strategy << std::endl;
149 : 0 : }
150 : : );
151 : 1 : C.arg_parser().add<const char*>(
152 : : /* group= */ "Wasm",
153 : : /* short= */ nullptr,
154 : : /* long= */ "--index-scan-materialization-strategy",
155 : : /* description= */ "specify the materialization strategy for index scans (`Inline` or `Memory`)",
156 : 0 : /* callback= */ [](const char *strategy){
157 [ # # ]: 0 : if (streq(strategy, "Inline"))
158 : 0 : options::index_scan_materialization_strategy = option_configs::IndexScanMaterializationStrategy::INLINE;
159 [ # # ]: 0 : else if (streq(strategy, "Memory"))
160 : 0 : options::index_scan_materialization_strategy = option_configs::IndexScanMaterializationStrategy::MEMORY;
161 : : else
162 : 0 : std::cerr << "warning: ignore invalid index scan strategy " << strategy << std::endl;
163 : 0 : }
164 : : );
165 : 1 : C.arg_parser().add<const char*>(
166 : : /* group= */ "Wasm",
167 : : /* short= */ nullptr,
168 : : /* long= */ "--filter-selection-strategy",
169 : : /* description= */ "specify the selection strategy for filters (`Branching` or `Predicated`)",
170 : 0 : /* callback= */ [](const char *strategy){
171 [ # # ]: 0 : if (streq(strategy, "Branching"))
172 : 0 : options::filter_selection_strategy = option_configs::SelectionStrategy::BRANCHING;
173 [ # # ]: 0 : else if (streq(strategy, "Predicated"))
174 : 0 : options::filter_selection_strategy = option_configs::SelectionStrategy::PREDICATED;
175 : : else
176 : 0 : std::cerr << "warning: ignore invalid filter selection strategy " << strategy << std::endl;
177 : 0 : }
178 : : );
179 : 1 : C.arg_parser().add<const char*>(
180 : : /* group= */ "Wasm",
181 : : /* short= */ nullptr,
182 : : /* long= */ "--quicksort-cmp-selection-strategy",
183 : : /* description= */ "specify the selection strategy for comparisons in quicksort (`Branching` or `Predicated`)",
184 : 0 : /* callback= */ [](const char *strategy){
185 [ # # ]: 0 : if (streq(strategy, "Branching"))
186 : 0 : options::quicksort_cmp_selection_strategy = option_configs::SelectionStrategy::BRANCHING;
187 [ # # ]: 0 : else if (streq(strategy, "Predicated"))
188 : 0 : options::quicksort_cmp_selection_strategy = option_configs::SelectionStrategy::PREDICATED;
189 : : else
190 : 0 : std::cerr << "warning: ignore invalid quicksort comparison selection strategy " << strategy << std::endl;
191 : 0 : }
192 [ # # ]: 0 : );
193 : 1 : C.arg_parser().add<const char*>(
194 : : /* group= */ "Wasm",
195 : : /* short= */ nullptr,
196 : : /* long= */ "--nested-loops-join-selection-strategy",
197 : : /* description= */ "specify the selection strategy for nested-loops joins (`Branching` or `Predicated`)",
198 : 0 : /* callback= */ [](const char *strategy){
199 [ # # ]: 0 : if (streq(strategy, "Branching"))
200 [ # # ]: 0 : options::nested_loops_join_selection_strategy = option_configs::SelectionStrategy::BRANCHING;
201 [ # # ]: 0 : else if (streq(strategy, "Predicated"))
202 : 0 : options::nested_loops_join_selection_strategy = option_configs::SelectionStrategy::PREDICATED;
203 : : else
204 : 0 : std::cerr << "warning: ignore invalid nested-loops join selection strategy " << strategy << std::endl;
205 : 0 : }
206 : : );
207 : 1 : C.arg_parser().add<const char*>(
208 : : /* group= */ "Wasm",
209 : : /* short= */ nullptr,
210 : : /* long= */ "--simple-hash-join-selection-strategy",
211 : : /* description= */ "specify the selection strategy for simple hash joins (`Branching` or `Predicated`)",
212 : 0 : /* callback= */ [](const char *strategy){
213 [ # # ]: 0 : if (streq(strategy, "Branching"))
214 : 0 : options::simple_hash_join_selection_strategy = option_configs::SelectionStrategy::BRANCHING;
215 [ # # ]: 0 : else if (streq(strategy, "Predicated"))
216 : 0 : options::simple_hash_join_selection_strategy = option_configs::SelectionStrategy::PREDICATED;
217 : : else
218 : 0 : std::cerr << "warning: ignore invalid simple hash join selection strategy " << strategy << std::endl;
219 : 0 : }
220 : : );
221 : 1 : C.arg_parser().add<const char*>(
222 : : /* group= */ "Wasm",
223 : : /* short= */ nullptr,
224 : : /* long= */ "--simple-hash-join-ordering-strategy",
225 : : /* description= */ "specify the ordering strategy for simple hash joins (`BuildOnLeft` or `BuildOnRight`)",
226 : 0 : /* callback= */ [](const char *strategy){
227 [ # # ]: 0 : if (streq(strategy, "BuildOnLeft"))
228 : 0 : options::simple_hash_join_ordering_strategy = option_configs::OrderingStrategy::BUILD_ON_LEFT;
229 [ # # ]: 0 : else if (streq(strategy, "BuildOnRight"))
230 : 0 : options::simple_hash_join_ordering_strategy = option_configs::OrderingStrategy::BUILD_ON_RIGHT;
231 : : else
232 : 0 : std::cerr << "warning: ignore invalid simple hash join ordering strategy " << strategy << std::endl;
233 : 0 : }
234 : : );
235 : 1 : C.arg_parser().add<const char*>(
236 : : /* group= */ "Wasm",
237 : : /* short= */ nullptr,
238 : : /* long= */ "--sort-merge-join-selection-strategy",
239 : : /* description= */ "specify the selection strategy for sort merge joins (`Branching` or `Predicated`)",
240 : 0 : /* callback= */ [](const char *strategy){
241 [ # # ]: 0 : if (streq(strategy, "Branching"))
242 : 0 : options::sort_merge_join_selection_strategy = option_configs::SelectionStrategy::BRANCHING;
243 [ # # ]: 0 : else if (streq(strategy, "Predicated"))
244 : 0 : options::sort_merge_join_selection_strategy = option_configs::SelectionStrategy::PREDICATED;
245 : : else
246 : 0 : std::cerr << "warning: ignore invalid sort merge join selection strategy " << strategy << std::endl;
247 : 0 : }
248 : : );
249 : 1 : C.arg_parser().add<const char*>(
250 : : /* group= */ "Wasm",
251 : : /* short= */ nullptr,
252 : : /* long= */ "--sort-merge-join-cmp-selection-strategy",
253 : : /* description= */ "specify the selection strategy for comparisons while sorting in sort merge joins "
254 : : "(`Branching` or `Predicated`)",
255 : 0 : /* callback= */ [](const char *strategy){
256 [ # # ]: 0 : if (streq(strategy, "Branching"))
257 : 0 : options::sort_merge_join_cmp_selection_strategy = option_configs::SelectionStrategy::BRANCHING;
258 [ # # ]: 0 : else if (streq(strategy, "Predicated"))
259 : 0 : options::sort_merge_join_cmp_selection_strategy = option_configs::SelectionStrategy::PREDICATED;
260 : : else
261 : 0 : std::cerr << "warning: ignore invalid sort merge join comparison selection strategy " << strategy
262 : 0 : << std::endl;
263 : 0 : }
264 : : );
265 : 1 : C.arg_parser().add<const char*>(
266 : : /* group= */ "Wasm",
267 : : /* short= */ nullptr,
268 : : /* long= */ "--hash-table-implementation",
269 : : /* description= */ "specify the hash table implementation (`OpenAddressing` or `Chained`)",
270 : 0 : /* callback= */ [](const char *impl){
271 [ # # ]: 0 : if (streq(impl, "OpenAddressing"))
272 : 0 : options::hash_table_implementation = option_configs::HashTableImplementation::OPEN_ADDRESSING;
273 [ # # ]: 0 : else if (streq(impl, "Chained"))
274 : 0 : options::hash_table_implementation = option_configs::HashTableImplementation::CHAINED;
275 : : else
276 : 0 : std::cerr << "warning: ignore invalid hash table implementation " << impl << std::endl;
277 : 0 : }
278 : : );
279 : 1 : C.arg_parser().add<const char*>(
280 : : /* group= */ "Wasm",
281 : : /* short= */ nullptr,
282 : : /* long= */ "--hash-table-probing-strategy",
283 : : /* description= */ "specify the probing strategy for hash tables (`Linear` or `Quadratic`)",
284 : 0 : /* callback= */ [](const char *strategy){
285 [ # # ]: 0 : if (streq(strategy, "Linear"))
286 : 0 : options::hash_table_probing_strategy = option_configs::ProbingStrategy::LINEAR;
287 [ # # ]: 0 : else if (streq(strategy, "Quadratic"))
288 : 0 : options::hash_table_probing_strategy = option_configs::ProbingStrategy::QUADRATIC;
289 : : else
290 : 0 : std::cerr << "warning: ignore invalid hash table probing strategy " << strategy << std::endl;
291 : 0 : }
292 : : );
293 : 1 : C.arg_parser().add<const char*>(
294 : : /* group= */ "Wasm",
295 : : /* short= */ nullptr,
296 : : /* long= */ "--hash-table-storing-strategy",
297 : : /* description= */ "specify the storing strategy for hash tables (`InPlace` or `OutOfPlace`)",
298 : 0 : /* callback= */ [](const char *strategy){
299 [ # # ]: 0 : if (streq(strategy, "InPlace"))
300 : 0 : options::hash_table_storing_strategy = option_configs::StoringStrategy::IN_PLACE;
301 [ # # ]: 0 : else if (streq(strategy, "OutOfPlace"))
302 : 0 : options::hash_table_storing_strategy = option_configs::StoringStrategy::OUT_OF_PLACE;
303 : : else
304 : 0 : std::cerr << "warning: ignore invalid hash table storing strategy " << strategy << std::endl;
305 : 0 : }
306 : : );
307 : 1 : C.arg_parser().add<double>(
308 : : /* group= */ "Wasm",
309 : : /* short= */ nullptr,
310 : : /* long= */ "--hash-table-max-load-factor",
311 : : /* description= */ "specify the maximal load factor for hash tables, i.e. the load factor at which rehashing "
312 : : "should occur (must be in [1,∞) for chained and in [0.5,1) for open-addressing hash tables)",
313 : 0 : /* callback= */ [](double load_factor){
314 : 0 : options::load_factor_open_addressing = load_factor;
315 : 0 : options::load_factor_chained = load_factor;
316 : 0 : }
317 : : );
318 : 1 : C.arg_parser().add<double>(
319 : : /* group= */ "Wasm",
320 : : /* short= */ nullptr,
321 : : /* long= */ "--hash-table-initial-capacity",
322 : : /* description= */ "specify the initial capacity for hash tables",
323 : 0 : /* callback= */ [](uint32_t initial_capacity){
324 : 0 : options::hash_table_initial_capacity = initial_capacity;
325 : 0 : }
326 : : );
327 : 1 : C.arg_parser().add<bool>(
328 : : /* group= */ "Wasm",
329 : : /* short= */ nullptr,
330 : : /* long= */ "--no-hash-based-group-join",
331 : : /* description= */ "disable potential use of hash-based group-join",
332 : 0 : /* callback= */ [](bool){ options::hash_based_group_join = false; }
333 : : );
334 : 1 : C.arg_parser().add<const char*>(
335 : : /* group= */ "Wasm",
336 : : /* short= */ nullptr,
337 : : /* long= */ "--hard-pipeline-breaker-layout",
338 : : /* description= */ "specify the layout for hard pipeline breakers (`Row`, `PAX4K`, `PAX64K`, `PAX512K`, "
339 : : "`PAX4M`, or `PAX64M`)",
340 : 0 : /* callback= */ [](const char *layout){
341 [ # # ]: 0 : if (streq(layout, "Row")) {
342 : 0 : options::hard_pipeline_breaker_layout = std::make_unique<RowLayoutFactory>();
343 : 0 : } else {
344 : : PAXLayoutFactory::block_size_t size_type;
345 : : uint64_t block_size;
346 [ # # ]: 0 : if (streq(layout, "PAX4K")) {
347 : 0 : size_type = PAXLayoutFactory::NBytes;
348 : 0 : block_size = 1UL << 12;
349 [ # # ]: 0 : } else if (streq(layout, "PAX64K")) {
350 : 0 : size_type = PAXLayoutFactory::NBytes;
351 : 0 : block_size = 1UL << 16;
352 [ # # ]: 0 : } else if (streq(layout, "PAX512K")) {
353 : 0 : size_type = PAXLayoutFactory::NBytes;
354 : 0 : block_size = 1UL << 19;
355 [ # # ]: 0 : } else if (streq(layout, "PAX4M")) {
356 : 0 : size_type = PAXLayoutFactory::NBytes;
357 : 0 : block_size = 1UL << 22;
358 [ # # ]: 0 : } else if (streq(layout, "PAX64M")) {
359 : 0 : size_type = PAXLayoutFactory::NBytes;
360 : 0 : block_size = 1UL << 26;
361 [ # # ]: 0 : } else if (streq(layout, "PAX16Tup")) {
362 : 0 : size_type = PAXLayoutFactory::NTuples;
363 : 0 : block_size = 16;
364 [ # # ]: 0 : } else if (streq(layout, "PAX128Tup")) {
365 : 0 : size_type = PAXLayoutFactory::NTuples;
366 : 0 : block_size = 128;
367 [ # # ]: 0 : } else if (streq(layout, "PAX1024Tup")) {
368 : 0 : size_type = PAXLayoutFactory::NTuples;
369 : 0 : block_size = 1024;
370 : 0 : } else {
371 : 0 : std::cerr << "warning: ignore invalid layout for hard pipeline breakers " << layout << std::endl;
372 : : }
373 : 0 : options::hard_pipeline_breaker_layout = std::make_unique<PAXLayoutFactory>(size_type, block_size);
374 : : }
375 : 0 : }
376 : : );
377 : 1 : C.arg_parser().add<std::vector<std::string_view>>(
378 : : /* group= */ "Wasm",
379 : : /* short= */ nullptr,
380 : : /* long= */ "--soft-pipeline-breaker",
381 : : /* description= */ "a comma seperated list where to insert soft pipeline breakers (`AfterAll`, `AfterScan`, "
382 : : "`AfterFilter`, `AfterProjection`, `AfterNestedLoopsJoin`, or `AfterSimpleHashJoin`)",
383 : 0 : /* callback= */ [](std::vector<std::string_view> location){
384 : 0 : options::soft_pipeline_breaker = option_configs::SoftPipelineBreakerStrategy(0UL);
385 [ # # ]: 0 : for (const auto &elem : location) {
386 [ # # ]: 0 : if (strneq(elem.data(), "AfterAll", elem.size()))
387 : 0 : options::soft_pipeline_breaker |= option_configs::SoftPipelineBreakerStrategy::AFTER_ALL;
388 [ # # ]: 0 : else if (strneq(elem.data(), "AfterScan", elem.size()))
389 : 0 : options::soft_pipeline_breaker |= option_configs::SoftPipelineBreakerStrategy::AFTER_SCAN;
390 [ # # ]: 0 : else if (strneq(elem.data(), "AfterFilter", elem.size()))
391 : 0 : options::soft_pipeline_breaker |= option_configs::SoftPipelineBreakerStrategy::AFTER_FILTER;
392 [ # # ]: 0 : else if (strneq(elem.data(), "AfterIndexScan", elem.size()))
393 : 0 : options::soft_pipeline_breaker |= option_configs::SoftPipelineBreakerStrategy::AFTER_INDEX_SCAN;
394 [ # # ]: 0 : else if (strneq(elem.data(), "AfterProjection", elem.size()))
395 : 0 : options::soft_pipeline_breaker |= option_configs::SoftPipelineBreakerStrategy::AFTER_PROJECTION;
396 [ # # ]: 0 : else if (strneq(elem.data(), "AfterNestedLoopsJoin", elem.size()))
397 : 0 : options::soft_pipeline_breaker |= option_configs::SoftPipelineBreakerStrategy::AFTER_NESTED_LOOPS_JOIN;
398 [ # # ]: 0 : else if (strneq(elem.data(), "AfterSimpleHashJoin", elem.size()))
399 : 0 : options::soft_pipeline_breaker |= option_configs::SoftPipelineBreakerStrategy::AFTER_SIMPLE_HASH_JOIN;
400 : : else
401 : 0 : std::cerr << "warning: ignore invalid location for soft pipeline breakers " << elem << std::endl;
402 : : }
403 : 0 : }
404 : : );
405 : 1 : C.arg_parser().add<const char*>(
406 : : /* group= */ "Wasm",
407 : : /* short= */ nullptr,
408 : : /* long= */ "--soft-pipeline-breaker-layout",
409 : : /* description= */ "specify the layout for soft pipeline breakers (`Row`, `PAX4K`, `PAX64K`, `PAX512K`, "
410 : : "`PAX4M`, or `PAX64M`)",
411 : 0 : /* callback= */ [](const char *layout){
412 [ # # ]: 0 : if (streq(layout, "Row")) {
413 : 0 : options::soft_pipeline_breaker_layout = std::make_unique<RowLayoutFactory>();
414 : 0 : } else {
415 : : PAXLayoutFactory::block_size_t size_type;
416 : : uint64_t block_size;
417 [ # # ]: 0 : if (streq(layout, "PAX4K")) {
418 : 0 : size_type = PAXLayoutFactory::NBytes;
419 : 0 : block_size = 1UL << 12;
420 [ # # ]: 0 : } else if (streq(layout, "PAX64K")) {
421 : 0 : size_type = PAXLayoutFactory::NBytes;
422 : 0 : block_size = 1UL << 16;
423 [ # # ]: 0 : } else if (streq(layout, "PAX512K")) {
424 : 0 : size_type = PAXLayoutFactory::NBytes;
425 : 0 : block_size = 1UL << 19;
426 [ # # ]: 0 : } else if (streq(layout, "PAX4M")) {
427 : 0 : size_type = PAXLayoutFactory::NBytes;
428 : 0 : block_size = 1UL << 22;
429 [ # # ]: 0 : } else if (streq(layout, "PAX64M")) {
430 : 0 : size_type = PAXLayoutFactory::NBytes;
431 : 0 : block_size = 1UL << 26;
432 [ # # ]: 0 : } else if (streq(layout, "PAX16Tup")) {
433 : 0 : size_type = PAXLayoutFactory::NTuples;
434 : 0 : block_size = 16;
435 [ # # ]: 0 : } else if (streq(layout, "PAX128Tup")) {
436 : 0 : size_type = PAXLayoutFactory::NTuples;
437 : 0 : block_size = 128;
438 [ # # ]: 0 : } else if (streq(layout, "PAX1024Tup")) {
439 : 0 : size_type = PAXLayoutFactory::NTuples;
440 : 0 : block_size = 1024;
441 : 0 : } else {
442 : 0 : std::cerr << "warning: ignore invalid layout for soft pipeline breakers " << layout << std::endl;
443 : : }
444 : 0 : options::soft_pipeline_breaker_layout = std::make_unique<PAXLayoutFactory>(size_type, block_size);
445 : : }
446 : 0 : }
447 : : );
448 : 1 : C.arg_parser().add<std::size_t>(
449 : : /* group= */ "Wasm",
450 : : /* short= */ nullptr,
451 : : /* long= */ "--soft-pipeline-breaker-num-tuples",
452 : : /* description= */ "set the size in tuples for soft pipeline breakers (0 means infinite)",
453 : 0 : /* callback= */ [](std::size_t num_tuples){ options::soft_pipeline_breaker_num_tuples = num_tuples; }
454 : : );
455 : 1 : C.arg_parser().add<std::size_t>(
456 : : /* group= */ "Wasm",
457 : : /* short= */ nullptr,
458 : : /* long= */ "--index-sequential-scan-batch-size",
459 : : /* description= */ "set the number of tuples ids communicated between host and V8 per batch during index "
460 : : "sequential scan"
461 : : "(0 means infinite), ignored in case of --isam-compile-qualifying",
462 : 0 : /* callback= */ [](std::size_t size){ options::index_sequential_scan_batch_size = size; }
463 : : );
464 : 1 : C.arg_parser().add<std::size_t>(
465 : : /* group= */ "Wasm",
466 : : /* short= */ nullptr,
467 : : /* long= */ "--result-set-window-size",
468 : : /* description= */ "set the window size in tuples for the result set (0 means infinite)",
469 : 0 : /* callback= */ [](std::size_t size){ options::result_set_window_size = size; }
470 : : );
471 : 1 : C.arg_parser().add<bool>(
472 : : /* group= */ "Wasm",
473 : : /* short= */ nullptr,
474 : : /* long= */ "--no-exploit-unique-build",
475 : : /* description= */ "disable potential exploitation of uniqueness of build key in hash joins",
476 : 0 : /* callback= */ [](bool){ options::exploit_unique_build = false; }
477 : : );
478 : 1 : C.arg_parser().add<bool>(
479 : : /* group= */ "Wasm",
480 : : /* short= */ nullptr,
481 : : /* long= */ "--no-simd",
482 : : /* description= */ "disable potential use of SIMDfication",
483 : 0 : /* callback= */ [](bool){ options::simd = false; }
484 : : );
485 : 1 : C.arg_parser().add<bool>(
486 : : /* group= */ "Wasm",
487 : : /* short= */ nullptr,
488 : : /* long= */ "--no-double-pumping",
489 : : /* description= */ "disable use of double pumping (has only an effect if SIMDfication is enabled)",
490 : 0 : /* callback= */ [](bool){ options::double_pumping = false; }
491 : : );
492 : 1 : C.arg_parser().add<std::size_t>(
493 : : /* group= */ "Wasm",
494 : : /* short= */ nullptr,
495 : : /* long= */ "--simd-lanes",
496 : : /* description= */ "set the number of SIMD lanes to prefer",
497 : 0 : /* callback= */ [](std::size_t lanes){ options::simd_lanes = lanes; }
498 : : );
499 : 2 : C.arg_parser().add<std::vector<std::string_view>>(
500 : : /* group= */ "Hacks",
501 : : /* short= */ nullptr,
502 : : /* long= */ "--xxx-asc-sorted-attributes",
503 : : /* description= */ "a comma seperated list of attributes, i.e. of the format `T.x` where `T` is either the "
504 : : "table name or the alias and `x` is the attribute name, which are assumed to be sorted "
505 : : "ascending",
506 : 1 : /* callback= */ [&C](std::vector<std::string_view> attrs){
507 [ # # ]: 0 : for (const auto &elem : attrs) {
508 : 0 : auto idx = elem.find('.');
509 [ # # ]: 0 : if (idx == std::string_view::npos)
510 : 0 : std::cerr << "warning: ignore invalid attribute " << elem << std::endl;
511 [ # # # # : 0 : Schema::Identifier attr(C.pool(elem.substr(0, idx)), C.pool(elem.substr(idx + 1)));
# # # # ]
512 [ # # ]: 0 : options::sorted_attributes.emplace_back(std::move(attr), true);
513 : 0 : }
514 : 0 : }
515 : : );
516 : 2 : C.arg_parser().add<std::vector<std::string_view>>(
517 : : /* group= */ "Hacks",
518 : : /* short= */ nullptr,
519 : : /* long= */ "--xxx-desc-sorted-attributes",
520 : : /* description= */ "a comma seperated list of attributes, i.e. of the format `T.x` where `T` is either the "
521 : : "table name or the alias and `x` is the attribute name, which are assumed to be sorted "
522 : : "descending",
523 : 1 : /* callback= */ [&C](std::vector<std::string_view> attrs){
524 [ # # ]: 0 : for (const auto &elem : attrs) {
525 : 0 : auto idx = elem.find('.');
526 [ # # ]: 0 : if (idx == std::string_view::npos)
527 : 0 : std::cerr << "warning: ignore invalid attribute " << elem << std::endl;
528 [ # # # # : 0 : Schema::Identifier attr(C.pool(elem.substr(0, idx)), C.pool(elem.substr(idx + 1)));
# # # # ]
529 [ # # ]: 0 : options::sorted_attributes.emplace_back(std::move(attr), false);
530 : 0 : }
531 : 0 : }
532 : : );
533 : 1 : }
534 : :
535 : : }
536 : :
537 : :
538 : : /*======================================================================================================================
539 : : * register_wasm_operators()
540 : : *====================================================================================================================*/
541 : :
542 : 0 : void m::register_wasm_operators(PhysicalOptimizer &phys_opt)
543 : : {
544 : 0 : phys_opt.register_operator<NoOp>();
545 : 0 : phys_opt.register_operator<Callback<false>>();
546 : 0 : phys_opt.register_operator<Callback<true>>();
547 : 0 : phys_opt.register_operator<Print<false>>();
548 : 0 : phys_opt.register_operator<Print<true>>();
549 [ # # ]: 0 : if (bool(options::scan_implementations bitand option_configs::ScanImplementation::SCAN)) {
550 : 0 : phys_opt.register_operator<Scan<false>>();
551 [ # # ]: 0 : if (options::simd)
552 : 0 : phys_opt.register_operator<Scan<true>>();
553 : 0 : }
554 [ # # ]: 0 : if (bool(options::scan_implementations bitand option_configs::ScanImplementation::INDEX_SCAN)) {
555 [ # # ]: 0 : if (bool(options::index_implementations bitand option_configs::IndexImplementation::ARRAY))
556 : 0 : phys_opt.register_operator<IndexScan<idx::IndexMethod::Array>>();
557 [ # # ]: 0 : if (bool(options::index_implementations bitand option_configs::IndexImplementation::RMI))
558 : 0 : phys_opt.register_operator<IndexScan<idx::IndexMethod::Rmi>>();
559 : 0 : }
560 [ # # ]: 0 : if (bool(options::filter_selection_strategy bitand option_configs::SelectionStrategy::BRANCHING))
561 : 0 : phys_opt.register_operator<Filter<false>>();
562 [ # # ]: 0 : if (bool(options::filter_selection_strategy bitand option_configs::SelectionStrategy::PREDICATED))
563 : 0 : phys_opt.register_operator<Filter<true>>();
564 : 0 : phys_opt.register_operator<LazyDisjunctiveFilter>();
565 : 0 : phys_opt.register_operator<Projection>();
566 [ # # ]: 0 : if (bool(options::grouping_implementations bitand option_configs::GroupingImplementation::HASH_BASED))
567 : 0 : phys_opt.register_operator<HashBasedGrouping>();
568 [ # # ]: 0 : if (bool(options::grouping_implementations bitand option_configs::GroupingImplementation::ORDERED))
569 : 0 : phys_opt.register_operator<OrderedGrouping>();
570 : 0 : phys_opt.register_operator<Aggregation>();
571 [ # # ]: 0 : if (bool(options::sorting_implementations bitand option_configs::SortingImplementation::QUICKSORT)) {
572 [ # # ]: 0 : if (bool(options::quicksort_cmp_selection_strategy bitand option_configs::SelectionStrategy::BRANCHING))
573 : 0 : phys_opt.register_operator<Quicksort<false>>();
574 [ # # ]: 0 : if (bool(options::quicksort_cmp_selection_strategy bitand option_configs::SelectionStrategy::PREDICATED))
575 : 0 : phys_opt.register_operator<Quicksort<true>>();
576 : 0 : }
577 [ # # ]: 0 : if (bool(options::sorting_implementations bitand option_configs::SortingImplementation::NOOP))
578 : 0 : phys_opt.register_operator<NoOpSorting>();
579 [ # # ]: 0 : if (bool(options::join_implementations bitand option_configs::JoinImplementation::NESTED_LOOPS)) {
580 [ # # ]: 0 : if (bool(options::nested_loops_join_selection_strategy bitand option_configs::SelectionStrategy::BRANCHING))
581 : 0 : phys_opt.register_operator<NestedLoopsJoin<false>>();
582 [ # # ]: 0 : if (bool(options::nested_loops_join_selection_strategy bitand option_configs::SelectionStrategy::PREDICATED))
583 : 0 : phys_opt.register_operator<NestedLoopsJoin<true>>();
584 : 0 : }
585 [ # # ]: 0 : if (bool(options::join_implementations bitand option_configs::JoinImplementation::SIMPLE_HASH)) {
586 [ # # ]: 0 : if (bool(options::simple_hash_join_selection_strategy bitand option_configs::SelectionStrategy::BRANCHING)) {
587 [ + - ]: 1 : phys_opt.register_operator<SimpleHashJoin<false, false>>();
588 [ # # ]: 0 : if (options::exploit_unique_build)
589 [ + - ]: 1 : phys_opt.register_operator<SimpleHashJoin<true, false>>();
590 [ + - ]: 1 : }
591 [ # # ]: 0 : if (bool(options::simple_hash_join_selection_strategy bitand option_configs::SelectionStrategy::PREDICATED)) {
592 : 0 : phys_opt.register_operator<SimpleHashJoin<false, true>>();
593 [ # # ]: 0 : if (options::exploit_unique_build)
594 : 0 : phys_opt.register_operator<SimpleHashJoin<true, true>>();
595 : 0 : }
596 : 0 : }
597 [ # # ]: 0 : if (bool(options::join_implementations bitand option_configs::JoinImplementation::SORT_MERGE)) {
598 [ # # ]: 0 : if (bool(options::sort_merge_join_selection_strategy bitand option_configs::SelectionStrategy::BRANCHING)) {
599 [ # # ]: 0 : if (bool(options::sort_merge_join_cmp_selection_strategy bitand option_configs::SelectionStrategy::BRANCHING)) {
600 : 0 : phys_opt.register_operator<SortMergeJoin<false, false, false, false>>();
601 : 0 : phys_opt.register_operator<SortMergeJoin<false, true, false, false>>();
602 : 0 : phys_opt.register_operator<SortMergeJoin<true, false, false, false>>();
603 : 0 : phys_opt.register_operator<SortMergeJoin<true, true, false, false>>();
604 : 0 : }
605 [ # # ]: 0 : if (bool(options::sort_merge_join_cmp_selection_strategy bitand option_configs::SelectionStrategy::PREDICATED)) {
606 : 0 : phys_opt.register_operator<SortMergeJoin<false, false, false, true>>();
607 : 0 : phys_opt.register_operator<SortMergeJoin<false, true, false, true>>();
608 : 0 : phys_opt.register_operator<SortMergeJoin<true, false, false, true>>();
609 : 0 : phys_opt.register_operator<SortMergeJoin<true, true, false, true>>();
610 : 0 : }
611 : 0 : }
612 [ # # ]: 0 : if (bool(options::sort_merge_join_selection_strategy bitand option_configs::SelectionStrategy::PREDICATED)) {
613 [ # # ]: 0 : if (bool(options::sort_merge_join_cmp_selection_strategy bitand option_configs::SelectionStrategy::BRANCHING)) {
614 : 0 : phys_opt.register_operator<SortMergeJoin<false, false, true, false>>();
615 : 0 : phys_opt.register_operator<SortMergeJoin<false, true, true, false>>();
616 : 0 : phys_opt.register_operator<SortMergeJoin<true, false, true, false>>();
617 : 0 : phys_opt.register_operator<SortMergeJoin<true, true, true, false>>();
618 : 0 : }
619 [ # # ]: 0 : if (bool(options::sort_merge_join_cmp_selection_strategy bitand option_configs::SelectionStrategy::PREDICATED)) {
620 : 0 : phys_opt.register_operator<SortMergeJoin<false, false, true, true>>();
621 : 0 : phys_opt.register_operator<SortMergeJoin<false, true, true, true>>();
622 : 0 : phys_opt.register_operator<SortMergeJoin<true, false, true, true>>();
623 : 0 : phys_opt.register_operator<SortMergeJoin<true, true, true, true>>();
624 : 0 : }
625 : 0 : }
626 : 0 : }
627 : 0 : phys_opt.register_operator<Limit>();
628 [ # # ]: 0 : if (options::hash_based_group_join)
629 : 0 : phys_opt.register_operator<HashBasedGroupJoin>();
630 : 0 : }
631 : :
632 : :
633 : : /*======================================================================================================================
634 : : * Helper structs and functions
635 : : *====================================================================================================================*/
636 : :
637 : : /** Emits code to write the result set of the `Schema` \p schema using the `DataLayout` created by \p factory. The
638 : : * result set is either materialized entirely (if \p window_size equals 0 indicating infinity) or only partially (if
639 : : * \p window_size does not equal 0 indicating the used batch size). To emit the code at the correct position, code
640 : : * generation is delegated to the child physical operator \p child. */
641 : 0 : void write_result_set(const Schema &schema, const DataLayoutFactory &factory, uint32_t window_size,
642 : : const m::wasm::MatchBase &child)
643 : : {
644 [ # # # # : 0 : M_insist(schema == schema.drop_constants().deduplicate(), "schema must not contain constants or duplicates");
# # ]
645 : 0 : M_insist(CodeGenContext::Get().env().empty(), "all environment entries must be used");
646 : :
647 : : /*----- Set data layout factory used for the result set. -----*/
648 : 0 : auto &context = WasmEngine::Get_Wasm_Context_By_ID(Module::ID());
649 : 0 : context.result_set_factory = factory.clone();
650 : :
651 [ # # ]: 0 : if (schema.num_entries() == 0) { // result set contains only NULL constants
652 [ # # ]: 0 : if (window_size != 0) { // i.e. result set is materialized only partially
653 : 0 : M_insist(window_size >= CodeGenContext::Get().num_simd_lanes());
654 : 0 : M_insist(window_size % CodeGenContext::Get().num_simd_lanes() == 0);
655 : :
656 : 0 : std::optional<Var<U32x1>> counter; ///< variable to *locally* count
657 : : ///> *global* counter backup since the following code may be called multiple times
658 [ # # ]: 0 : Global<U32x1> counter_backup; // default initialized to 0
659 : :
660 : : /*----- Create child function s.t. result set is extracted in case of returns (e.g. due to `Limit`). -----*/
661 [ # # # # : 0 : FUNCTION(child_pipeline, void(void))
# # # # ]
662 : : {
663 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
664 : :
665 [ # # ]: 0 : child.execute(
666 [ # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ counter.emplace(counter_backup); }),
667 : 0 : /* pipeline= */ [&](){
668 : 0 : M_insist(bool(counter));
669 : :
670 : : /*----- Increment tuple ID. -----*/
671 [ # # ]: 0 : if (auto &env = CodeGenContext::Get().env(); env.predicated()) {
672 : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1,
673 : : "SIMDfication with predication not supported");
674 [ # # # # : 0 : *counter += env.extract_predicate<_Boolx1>().is_true_and_not_null().to<uint32_t>();
# # ]
675 : 0 : } else {
676 : 0 : *counter += uint32_t(CodeGenContext::Get().num_simd_lanes());
677 : : }
678 : :
679 : : /*----- If window size is reached, update result size, extract current results, and reset tuple ID. */
680 [ # # ]: 0 : IF (*counter == window_size) {
681 [ # # ]: 0 : CodeGenContext::Get().inc_num_tuples(U32x1(window_size));
682 [ # # # # ]: 0 : Module::Get().emit_call<void>("read_result_set", Ptr<void>::Nullptr(), U32x1(window_size));
683 : 0 : *counter = 0U;
684 : 0 : };
685 : 0 : },
686 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){
687 : 0 : M_insist(bool(counter));
688 : 0 : counter_backup = *counter;
689 : 0 : counter.reset();
690 : 0 : })
691 : : );
692 : 0 : }
693 [ # # ]: 0 : child_pipeline(); // call child function
694 : :
695 : : /*----- Update number of result tuples. -----*/
696 [ # # # # : 0 : CodeGenContext::Get().inc_num_tuples(counter_backup);
# # ]
697 : :
698 : : /*----- Extract remaining results. -----*/
699 [ # # # # : 0 : Module::Get().emit_call<void>("read_result_set", Ptr<void>::Nullptr(), counter_backup.val());
# # # # ]
700 : 0 : } else { // i.e. result set is materialized entirely
701 : : /*----- Create child function s.t. result set is extracted in case of returns (e.g. due to `Limit`). -----*/
702 [ # # # # : 0 : FUNCTION(child_pipeline, void(void))
# # ]
703 : : {
704 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
705 : :
706 : 0 : std::optional<Var<U32x1>> num_tuples; ///< variable to *locally* count additional result tuples
707 : :
708 [ # # ]: 0 : child.execute(
709 [ # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){
710 [ # # ]: 0 : num_tuples.emplace(CodeGenContext::Get().num_tuples());
711 : 0 : }),
712 : 0 : /* pipeline= */ [&](){
713 : 0 : M_insist(bool(num_tuples));
714 [ # # ]: 0 : if (auto &env = CodeGenContext::Get().env(); env.predicated()) {
715 : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1,
716 : : "SIMDfication with predication not supported");
717 [ # # # # : 0 : *num_tuples += env.extract_predicate<_Boolx1>().is_true_and_not_null().to<uint32_t>();
# # ]
718 : 0 : } else {
719 : 0 : *num_tuples += uint32_t(CodeGenContext::Get().num_simd_lanes());
720 : : }
721 : 0 : },
722 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){
723 : 0 : M_insist(bool(num_tuples));
724 [ # # ]: 0 : CodeGenContext::Get().set_num_tuples(*num_tuples);
725 : 0 : num_tuples.reset();
726 : 0 : })
727 : : );
728 : 0 : }
729 [ # # ]: 0 : child_pipeline(); // call child function
730 : :
731 : : /*----- Extract all results at once. -----*/
732 [ # # # # : 0 : Module::Get().emit_call<void>("read_result_set", Ptr<void>::Nullptr(), CodeGenContext::Get().num_tuples());
# # # # #
# ]
733 : 0 : }
734 : 0 : } else { // result set contains contains actual values
735 [ # # ]: 0 : if (window_size != 0) { // i.e. result set is materialized only partially
736 : 0 : M_insist(window_size > CodeGenContext::Get().num_simd_lanes());
737 : 0 : M_insist(window_size % CodeGenContext::Get().num_simd_lanes() == 0);
738 : :
739 : : /*----- Create finite global buffer (without `pipeline`-callback) used as reusable result set. -----*/
740 [ # # # # : 0 : GlobalBuffer result_set(schema, factory, false, window_size); // no callback to extract windows manually
# # ]
741 : :
742 : : /*----- Create child function s.t. result set is extracted in case of returns (e.g. due to `Limit`). -----*/
743 [ # # # # : 0 : FUNCTION(child_pipeline, void(void))
# # # # ]
744 : : {
745 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
746 : :
747 [ # # ]: 0 : child.execute(
748 [ # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ result_set.setup(); }),
749 : 0 : /* pipeline= */ [&](){
750 : : /*----- Store whether only a single slot is free to not extract result for empty buffer. -----*/
751 [ # # ]: 0 : const Var<Boolx1> single_slot_free(
752 [ # # # # ]: 0 : result_set.size() == window_size - uint32_t(CodeGenContext::Get().num_simd_lanes())
753 : : );
754 : :
755 : : /*----- Write the result. -----*/
756 [ # # ]: 0 : result_set.consume(); // also resets size to 0 in case buffer has reached window size
757 : :
758 : : /*----- If the last buffer slot was filled, update result size and extract current results. */
759 [ # # # # : 0 : IF (single_slot_free and result_set.size() == 0U) {
# # # # ]
760 [ # # ]: 0 : CodeGenContext::Get().inc_num_tuples(U32x1(window_size));
761 [ # # ]: 0 : Module::Get().emit_call<void>("read_result_set", result_set.base_address(),
762 [ # # ]: 0 : U32x1(window_size));
763 : 0 : };
764 : 0 : },
765 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ result_set.teardown(); })
766 : : );
767 : 0 : }
768 [ # # ]: 0 : child_pipeline(); // call child function
769 : :
770 : : /*----- Update number of result tuples. -----*/
771 [ # # # # : 0 : CodeGenContext::Get().inc_num_tuples(result_set.size());
# # ]
772 : :
773 : : /*----- Extract remaining results. -----*/
774 [ # # # # : 0 : Module::Get().emit_call<void>("read_result_set", result_set.base_address(), result_set.size());
# # # # ]
775 : 0 : } else { // i.e. result set is materialized entirely
776 : : /*----- Create infinite global buffer (without `pipeline`-callback) used as single result set. -----*/
777 [ # # # # : 0 : GlobalBuffer result_set(schema, factory); // no callback to extract results all at once
# # ]
778 : :
779 : : /*----- Create child function s.t. result set is extracted in case of returns (e.g. due to `Limit`). -----*/
780 [ # # # # : 0 : FUNCTION(child_pipeline, void(void))
# # # # ]
781 : : {
782 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
783 : :
784 [ # # ]: 0 : child.execute(
785 [ # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ result_set.setup(); }),
786 : 0 : /* pipeline= */ [&](){ result_set.consume(); },
787 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ result_set.teardown(); })
788 : : );
789 : 0 : }
790 [ # # ]: 0 : child_pipeline(); // call child function
791 : :
792 : : /*----- Set number of result tuples. -----*/
793 [ # # # # : 0 : CodeGenContext::Get().inc_num_tuples(result_set.size()); // not inside child function due to predication
# # ]
794 : :
795 : : /*----- Extract all results at once. -----*/
796 [ # # # # : 0 : Module::Get().emit_call<void>("read_result_set", result_set.base_address(), result_set.size());
# # # # ]
797 : 0 : }
798 : : }
799 : 0 : }
800 : :
801 : : ///> helper struct for aggregates
802 : : struct aggregate_info_t
803 : : {
804 : : Schema::entry_type entry; ///< aggregate entry consisting of identifier, type, and constraints
805 : : m::Function::fnid_t fnid; ///< aggregate function
806 : : const std::vector<std::unique_ptr<ast::Expr>> &args; ///< aggregate arguments
807 : : };
808 : :
809 : : ///> helper struct for AVG aggregates
810 : 0 : struct avg_aggregate_info_t
811 : : {
812 : : Schema::Identifier running_count; ///< identifier of running count
813 : : Schema::Identifier sum; ///< potential identifier for sum (only set if AVG is computed once at the end)
814 : : bool compute_running_avg; ///< flag whether running AVG must be computed instead of one computation at the end
815 : : };
816 : :
817 : : /** Computes and returns information about the aggregates \p aggregates which are contained in the schema \p schema
818 : : * starting at offset \p aggregates_offset. The firstly returned element contains general information about each
819 : : * aggregates like its identifier, type, function type, and arguments. The secondly returned element contains
820 : : * additional information about each AVG aggregates like a flag to determine whether it can be computed using a
821 : : * running AVG or lazily at the end. Either way, the corresponding running count and an optional sum are contained
822 : : * in these elements, too. */
823 : : std::pair<std::vector<aggregate_info_t>, std::unordered_map<Schema::Identifier, avg_aggregate_info_t>>
824 : 0 : compute_aggregate_info(const std::vector<std::reference_wrapper<const FnApplicationExpr>> &aggregates,
825 : : const Schema &schema, std::size_t aggregates_offset = 0)
826 : : {
827 : 0 : std::vector<aggregate_info_t> aggregates_info;
828 : 0 : std::unordered_map<Schema::Identifier, avg_aggregate_info_t> avg_aggregates_info;
829 : :
830 [ # # # # ]: 0 : for (std::size_t i = aggregates_offset; i < schema.num_entries(); ++i) {
831 [ # # ]: 0 : auto &e = schema[i];
832 : :
833 : 0 : auto pred = [&e](const auto &info){ return info.entry.id == e.id; };
834 [ # # # # ]: 0 : if (auto it = std::find_if(aggregates_info.cbegin(), aggregates_info.cend(), pred); it != aggregates_info.cend())
835 : 0 : continue; // duplicated aggregate
836 : :
837 : 0 : auto &fn_expr = aggregates[i - aggregates_offset].get();
838 [ # # ]: 0 : auto &fn = fn_expr.get_function();
839 [ # # ]: 0 : M_insist(fn.kind == m::Function::FN_Aggregate, "not an aggregation function");
840 : :
841 [ # # ]: 0 : if (fn.fnid == m::Function::FN_AVG) {
842 [ # # ]: 0 : M_insist(fn_expr.args.size() == 1, "AVG aggregate function expects exactly one argument");
843 : :
844 : : /*----- Insert a suitable running count, i.e. COUNT over the argument of the AVG aggregate. -----*/
845 : 0 : auto pred = [&fn_expr](const auto &_fn_expr){
846 [ # # ]: 0 : M_insist(_fn_expr.get().get_function().fnid != m::Function::FN_COUNT or _fn_expr.get().args.size() <= 1,
847 : : "COUNT aggregate function expects exactly one argument");
848 [ # # ]: 0 : return _fn_expr.get().get_function().fnid == m::Function::FN_COUNT and
849 [ # # ]: 0 : not _fn_expr.get().args.empty() and *_fn_expr.get().args[0] == *fn_expr.args[0];
850 : : };
851 : 0 : std::optional<Schema::Identifier> running_count;
852 [ # # # # : 0 : if (auto it = std::find_if(aggregates.cbegin(), aggregates.cend(), pred);
# # ]
853 : 0 : it != aggregates.cend())
854 : : { // reuse found running count
855 [ # # ]: 0 : const auto idx_agg = std::distance(aggregates.cbegin(), it);
856 [ # # # # ]: 0 : running_count = schema[aggregates_offset + idx_agg].id;
857 : 0 : } else { // insert additional running count
858 [ # # ]: 0 : std::ostringstream oss;
859 [ # # # # ]: 0 : oss << "$running_count_" << fn_expr;
860 [ # # # # : 0 : running_count = Schema::Identifier(Catalog::Get().pool(oss.str().c_str()));
# # # # #
# ]
861 [ # # # # ]: 0 : aggregates_info.emplace_back(aggregate_info_t{
862 [ # # # # : 0 : .entry = { *running_count, Type::Get_Integer(Type::TY_Scalar, 8), Schema::entry_type::NOT_NULLABLE },
# # # # ]
863 : : .fnid = m::Function::FN_COUNT,
864 : 0 : .args = fn_expr.args
865 : : });
866 : 0 : }
867 : :
868 : : /*----- Decide how to compute the average aggregate and insert sum aggregate accordingly. -----*/
869 : 0 : std::optional<Schema::Identifier> sum;
870 : : bool compute_running_avg;
871 [ # # # # : 0 : if (fn_expr.args[0]->type()->size() <= 32) {
# # ]
872 : : /* Compute average by summing up all values in a 64-bit field (thus no overflows should occur) and
873 : : * dividing by the running count once at the end. */
874 : 0 : compute_running_avg = false;
875 : 0 : auto pred = [&fn_expr](const auto &_fn_expr){
876 [ # # ]: 0 : M_insist(_fn_expr.get().get_function().fnid != m::Function::FN_SUM or
877 : : _fn_expr.get().args.size() == 1,
878 : : "SUM aggregate function expects exactly one argument");
879 [ # # ]: 0 : return _fn_expr.get().get_function().fnid == m::Function::FN_SUM and
880 : 0 : *_fn_expr.get().args[0] == *fn_expr.args[0];
881 : : };
882 [ # # # # : 0 : if (auto it = std::find_if(aggregates.cbegin(), aggregates.cend(), pred);
# # ]
883 : 0 : it != aggregates.cend())
884 : : { // reuse found SUM aggregate
885 [ # # ]: 0 : const auto idx_agg = std::distance(aggregates.cbegin(), it);
886 [ # # # # ]: 0 : sum = schema[aggregates_offset + idx_agg].id;
887 : 0 : } else { // insert additional SUM aggregate
888 [ # # ]: 0 : std::ostringstream oss;
889 [ # # # # ]: 0 : oss << "$sum_" << fn_expr;
890 [ # # # # : 0 : sum = Schema::Identifier(Catalog::Get().pool(oss.str().c_str()));
# # # # #
# ]
891 : : const Type *type;
892 [ # # # # : 0 : switch (as<const Numeric>(*fn_expr.args[0]->type()).kind) {
# # ]
893 : : case Numeric::N_Int:
894 : : case Numeric::N_Decimal:
895 [ # # # # ]: 0 : type = Type::Get_Integer(Type::TY_Scalar, 8);
896 : 0 : break;
897 : : case Numeric::N_Float:
898 [ # # # # ]: 0 : type = Type::Get_Double(Type::TY_Scalar);
899 : 0 : }
900 [ # # # # ]: 0 : aggregates_info.emplace_back(aggregate_info_t{
901 [ # # # # ]: 0 : .entry = { *sum, type, e.constraints },
902 : : .fnid = m::Function::FN_SUM,
903 : 0 : .args = fn_expr.args
904 : : });
905 : 0 : }
906 : 0 : } else {
907 : : /* Compute average by computing a running average for each inserted value in a `_Doublex1` field (since
908 : : * the sum may overflow). */
909 : 0 : compute_running_avg = true;
910 [ # # # # ]: 0 : M_insist(e.type->is_double());
911 [ # # # # ]: 0 : aggregates_info.emplace_back(aggregate_info_t{
912 [ # # ]: 0 : .entry = e,
913 : : .fnid = m::Function::FN_AVG,
914 : 0 : .args = fn_expr.args
915 : : });
916 : : }
917 : :
918 : : /*----- Add info for this AVG aggregate. -----*/
919 [ # # # # ]: 0 : avg_aggregates_info.try_emplace(e.id, avg_aggregate_info_t{
920 [ # # ]: 0 : .running_count = std::move(*running_count),
921 [ # # ]: 0 : .sum = std::move(*sum),
922 : 0 : .compute_running_avg = compute_running_avg
923 : : });
924 : 0 : } else {
925 [ # # # # : 0 : aggregates_info.emplace_back(aggregate_info_t{
# # ]
926 [ # # ]: 0 : .entry = e,
927 : 0 : .fnid = fn.fnid,
928 : 0 : .args = fn_expr.args
929 : : });
930 : : }
931 : 0 : }
932 : :
933 : 0 : return { std::move(aggregates_info), std::move(avg_aggregates_info) };
934 : 0 : }
935 : :
936 : : /** Decompose the equi-predicate \p cnf, i.e. a conjunction of equality comparisons of each two designators, into all
937 : : * identifiers contained in schema \p schema_left (returned as first element) and all identifiers not contained in
938 : : * the aforementioned schema (return as second element). */
939 : : std::pair<std::vector<Schema::Identifier>, std::vector<Schema::Identifier>>
940 : 0 : decompose_equi_predicate(const cnf::CNF &cnf, const Schema &schema_left)
941 : : {
942 : 0 : std::vector<Schema::Identifier> ids_left, ids_right;
943 [ # # ]: 0 : for (auto &clause : cnf) {
944 [ # # ]: 0 : M_insist(clause.size() == 1, "invalid equi-predicate");
945 : 0 : auto &literal = clause[0];
946 [ # # # # ]: 0 : auto &binary = as<const BinaryExpr>(literal.expr());
947 [ # # # # : 0 : M_insist((not literal.negative() and binary.tok == TK_EQUAL) or
# # # # #
# # # #
# ]
948 : : (literal.negative() and binary.tok == TK_BANG_EQUAL), "invalid equi-predicate");
949 [ # # # # ]: 0 : M_insist(is<const Designator>(binary.lhs), "invalid equi-predicate");
950 [ # # # # ]: 0 : M_insist(is<const Designator>(binary.rhs), "invalid equi-predicate");
951 [ # # # # ]: 0 : Schema::Identifier id_first(*binary.lhs), id_second(*binary.rhs);
952 [ # # # # : 0 : const auto &[id_left, id_right] = schema_left.has(id_first) ? std::make_pair(id_first, id_second)
# # ]
953 [ # # ]: 0 : : std::make_pair(id_second, id_first);
954 [ # # ]: 0 : ids_left.push_back(std::move(id_left));
955 [ # # ]: 0 : ids_right.push_back(std::move(id_right));
956 : 0 : }
957 [ # # ]: 0 : M_insist(ids_left.size() == ids_right.size(), "number of found IDs differ");
958 [ # # ]: 0 : M_insist(not ids_left.empty(), "must find at least one ID");
959 : 0 : return { std::move(ids_left), std::move(ids_right) };
960 : 0 : }
961 : :
962 : : /** Returns the number of rows of table \p table_name. */
963 : 10 : U32x1 get_num_rows(const ThreadSafePooledString &table_name) {
964 [ + + - + : 10 : static std::ostringstream oss;
+ - ]
965 [ + - + - ]: 10 : oss.str("");
966 : 10 : oss << table_name << "_num_rows";
967 [ + - ]: 10 : return Module::Get().get_global<uint32_t>(oss.str().c_str());
968 : 0 : }
969 : :
970 : : /** Returns a pointer to the beginning of table \p table_name in the WebAssembly linear memory. */
971 : 10 : Ptr<void> get_base_address(const ThreadSafePooledString &table_name) {
972 [ + + - + : 10 : static std::ostringstream oss;
+ - ]
973 [ + - + - ]: 10 : oss.str("");
974 : 10 : oss << table_name << "_mem";
975 [ + - ]: 10 : return Module::Get().get_global<void*>(oss.str().c_str());
976 : 0 : }
977 : :
978 : : /** Computes the initial hash table capacity for \p op. The function ensures that the initial capacity is in the range
979 : : * [0, 2^32 - 1] such that the capacity does *not* exceed the `uint32_t` value limit. */
980 : 0 : uint32_t compute_initial_ht_capacity(const Operator &op, double load_factor) {
981 : : uint64_t initial_capacity;
982 [ # # ]: 0 : if (options::hash_table_initial_capacity) {
983 : 0 : initial_capacity = *options::hash_table_initial_capacity;
984 : 0 : } else {
985 [ # # ]: 0 : if (op.has_info())
986 : 0 : initial_capacity = static_cast<uint64_t>(std::ceil(op.info().estimated_cardinality / load_factor));
987 [ # # ]: 0 : else if (auto scan = cast<const ScanOperator>(&op))
988 : 0 : initial_capacity = static_cast<uint64_t>(std::ceil(scan->store().num_rows() / load_factor));
989 : : else
990 : 0 : initial_capacity = 1024; // fallback
991 : : }
992 [ # # ]: 0 : return std::in_range<uint32_t>(initial_capacity) ? initial_capacity : std::numeric_limits<uint32_t>::max();
993 : : }
994 : :
995 : : ///> helper struct holding the bounds for index scan
996 : 0 : struct index_scan_bounds_t
997 : : {
998 : 0 : Schema::entry_type attribute = Schema::entry_type::CreateArtificial(); ///< Attribute for which bounds should hold
999 : : std::optional<std::reference_wrapper<const ast::Expr>> lo, hi; ///< lo and hi bounds
1000 : : bool is_inclusive_lo, is_inclusive_hi; ///< flag to indicate if bounds are inclusive
1001 : : };
1002 : :
1003 : : /** Returns `true` iff \p expr is a valid bound. */
1004 : 0 : bool is_valid_bound(const ast::Expr &expr) {
1005 [ # # ]: 0 : if (is<const Constant>(expr))
1006 : 0 : return true;
1007 [ # # ]: 0 : if (auto u = cast<const UnaryExpr>(&expr)) {
1008 : : /* `UnaryExpr` must be followed by `Constant`. */
1009 [ # # # # : 0 : if ((u->op().type == TK_MINUS or u->op().type == TK_PLUS) and is<const Constant>(*u->expr))
# # # # #
# # # #
# ]
1010 : 0 : return true;
1011 : 0 : }
1012 : 0 : return false;
1013 : 0 : }
1014 : :
1015 : : /** Given an `Expr` \p expr representing a valid bound, returns a pair consiting of a constant and a boolean flag
1016 : : * indicating whether the constant is negative. */
1017 : 0 : std::pair<const Constant&, bool> get_valid_bound(const ast::Expr &expr) {
1018 : 0 : M_insist(is_valid_bound(expr), "bound must be valid");
1019 [ # # ]: 0 : if (auto c = cast<const Constant>(&expr))
1020 : 0 : return { *c, false };
1021 : 0 : auto &u = as<const UnaryExpr>(expr);
1022 : 0 : return { as<const Constant>(*u.expr), u.op().type == TK_MINUS };
1023 : 0 : }
1024 : :
1025 : : /** Extracts the bounds for performing index scan from CNF \p cnf. CNFs must either consist of
1026 : : * 1. a single equality predicate, e.g. "x = 42" or
1027 : : * 2. one or two predicates defining a (closed or open) range, e.g. "x > 42 AND x < 109" or "x < 42". */
1028 : 0 : index_scan_bounds_t extract_index_scan_bounds(const cnf::CNF &cnf)
1029 : : {
1030 : 0 : index_scan_bounds_t bounds;
1031 : :
1032 [ # # ]: 0 : M_insist(not cnf.empty(), "filter condition must not be empty");
1033 [ # # ]: 0 : auto designators = cnf.get_required();
1034 [ # # # # ]: 0 : M_insist(designators.num_entries() == 1, "filter condition must contain exactly one designator");
1035 [ # # # # ]: 0 : bounds.attribute = designators[0];
1036 : :
1037 [ # # ]: 0 : for (auto &clause : cnf) {
1038 [ # # ]: 0 : M_insist(clause.size() == 1, "invalid predicate");
1039 : 0 : auto &literal = clause[0];
1040 [ # # # # ]: 0 : auto &binary = as<const BinaryExpr>(literal.expr());
1041 [ # # # # : 0 : M_insist((is<const Designator>(binary.lhs) and is_valid_bound(*binary.rhs)) or
# # # # #
# # # #
# ]
1042 : : (is<const Designator>(binary.rhs) and is_valid_bound(*binary.lhs)), "invalid predicate");
1043 [ # # ]: 0 : bool has_attribute_left = is<const Designator>(binary.lhs);
1044 [ # # ]: 0 : auto &bound = has_attribute_left ? *binary.rhs : *binary.lhs;
1045 : :
1046 [ # # # # : 0 : switch(binary.tok.type) {
# # ]
1047 : : default:
1048 [ # # ]: 0 : M_unreachable("unsupported token type");
1049 : : case TK_EQUAL:
1050 [ # # # # ]: 0 : M_insist(not bool(bounds.lo) and not bool(bounds.hi), "bound already set");
1051 : 0 : bounds.lo = bounds.hi = std::cref(bound);
1052 : 0 : bounds.is_inclusive_lo = bounds.is_inclusive_hi = true;
1053 : 0 : break;
1054 : : case TK_GREATER:
1055 [ # # ]: 0 : if (has_attribute_left) {
1056 [ # # ]: 0 : M_insist(not bool(bounds.lo), "lo bound already set");
1057 : 0 : bounds.lo = std::cref(bound);
1058 : 0 : bounds.is_inclusive_lo = false;
1059 : 0 : } else {
1060 [ # # ]: 0 : M_insist(not bool(bounds.hi), "hi bound already set");
1061 : 0 : bounds.hi = std::cref(bound);
1062 : 0 : bounds.is_inclusive_hi = false;
1063 : : }
1064 : 0 : break;
1065 : : case TK_GREATER_EQUAL:
1066 [ # # ]: 0 : if (has_attribute_left) {
1067 [ # # ]: 0 : M_insist(not bool(bounds.lo), "lo bound already set");
1068 : 0 : bounds.lo = std::cref(bound);
1069 : 0 : bounds.is_inclusive_lo = true;
1070 : 0 : } else {
1071 [ # # ]: 0 : M_insist(not bool(bounds.hi), "hi bound already set");
1072 : 0 : bounds.hi = std::cref(bound);
1073 : 0 : bounds.is_inclusive_hi = true;
1074 : : }
1075 : 0 : break;
1076 : : case TK_LESS:
1077 [ # # ]: 0 : if (has_attribute_left) {
1078 [ # # ]: 0 : M_insist(not bool(bounds.hi), "hi bound already set");
1079 : 0 : bounds.hi = std::cref(bound);
1080 : 0 : bounds.is_inclusive_hi = false;
1081 : 0 : } else {
1082 [ # # ]: 0 : M_insist(not bool(bounds.lo), "lo bound already set");
1083 : 0 : bounds.lo = std::cref(bound);
1084 : 0 : bounds.is_inclusive_lo = false;
1085 : : }
1086 : 0 : break;
1087 : : case TK_LESS_EQUAL:
1088 [ # # ]: 0 : if (has_attribute_left) {
1089 [ # # ]: 0 : M_insist(not bool(bounds.hi), "hi bound already set");
1090 : 0 : bounds.hi = std::cref(bound);
1091 : 0 : bounds.is_inclusive_hi = true;
1092 : 0 : } else {
1093 [ # # ]: 0 : M_insist(not bool(bounds.lo), "lo bound already set");
1094 : 0 : bounds.lo = std::cref(bound);
1095 : 0 : bounds.is_inclusive_lo = true;
1096 : : }
1097 : 0 : break;
1098 : : }
1099 : : }
1100 [ # # # # ]: 0 : M_insist(bool(bounds.lo) or bool(bounds.hi), "either bound must be set");
1101 : 0 : return bounds;
1102 [ # # ]: 0 : }
1103 : :
1104 : :
1105 : : /*======================================================================================================================
1106 : : * NoOp
1107 : : *====================================================================================================================*/
1108 : :
1109 : 0 : void NoOp::execute(const Match<NoOp> &M, setup_t, pipeline_t, teardown_t)
1110 : : {
1111 : 0 : std::optional<Var<U32x1>> num_tuples; ///< variable to *locally* count additional result tuples
1112 : :
1113 [ # # ]: 0 : M.child->execute(
1114 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ num_tuples.emplace(CodeGenContext::Get().num_tuples()); }),
1115 : 0 : /* pipeline= */ [&](){
1116 : 0 : M_insist(bool(num_tuples));
1117 [ # # ]: 0 : if (auto &env = CodeGenContext::Get().env(); env.predicated()) {
1118 [ # # # ]: 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
1119 : 0 : default: M_unreachable("invalid number of simd lanes");
1120 : : case 1: {
1121 [ # # # # : 0 : *num_tuples += env.extract_predicate<_Boolx1>().is_true_and_not_null().to<uint32_t>();
# # ]
1122 : 0 : break;
1123 : : }
1124 : : case 16: {
1125 [ # # ]: 0 : auto pred = env.extract_predicate<_Boolx16>().is_true_and_not_null();
1126 [ # # # # : 0 : *num_tuples += pred.bitmask().popcnt();
# # ]
1127 : : break;
1128 : 0 : }
1129 : : }
1130 : 0 : } else {
1131 : 0 : *num_tuples += uint32_t(CodeGenContext::Get().num_simd_lanes());
1132 : : }
1133 : 0 : },
1134 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){
1135 : 0 : M_insist(bool(num_tuples));
1136 [ # # ]: 0 : CodeGenContext::Get().set_num_tuples(*num_tuples);
1137 : 0 : num_tuples.reset();
1138 : 0 : })
1139 : : );
1140 : 0 : }
1141 : :
1142 : :
1143 : : /*======================================================================================================================
1144 : : * Callback
1145 : : *====================================================================================================================*/
1146 : :
1147 : : template<bool SIMDfied>
1148 : 0 : ConditionSet Callback<SIMDfied>::pre_condition(std::size_t child_idx, const std::tuple<const CallbackOperator*>&)
1149 : : {
1150 : 0 : M_insist(child_idx == 0);
1151 : :
1152 : 0 : ConditionSet pre_cond;
1153 : :
1154 : : if constexpr (SIMDfied) {
1155 : : /*----- SIMDfied callback supports SIMD but not predication. -----*/
1156 [ # # # # ]: 0 : pre_cond.add_condition(Predicated(false));
1157 : : } else {
1158 : : /*----- Non-SIMDfied callback does not support SIMD. -----*/
1159 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
1160 : : }
1161 : :
1162 : 0 : return pre_cond;
1163 [ # # # # ]: 0 : }
1164 : :
1165 : : template<bool SIMDfied>
1166 : 0 : void Callback<SIMDfied>::execute(const Match<Callback> &M, setup_t, pipeline_t, teardown_t)
1167 : : {
1168 : 0 : M_insist(bool(M.result_set_factory), "`wasm::Callback` must have a factory for the result set");
1169 : :
1170 [ # # # # ]: 0 : auto result_set_schema = M.callback.schema().drop_constants().deduplicate();
1171 [ # # # # ]: 0 : write_result_set(result_set_schema, *M.result_set_factory, M.result_set_window_size, *M.child);
1172 : 0 : }
1173 : :
1174 : :
1175 : : /*======================================================================================================================
1176 : : * Print
1177 : : *====================================================================================================================*/
1178 : :
1179 : : template<bool SIMDfied>
1180 : 0 : ConditionSet Print<SIMDfied>::pre_condition(std::size_t child_idx, const std::tuple<const PrintOperator*>&)
1181 : : {
1182 : 0 : M_insist(child_idx == 0);
1183 : :
1184 : 0 : ConditionSet pre_cond;
1185 : :
1186 : : if constexpr (SIMDfied) {
1187 : : /*----- SIMDfied print supports SIMD but not predication. -----*/
1188 [ # # # # ]: 0 : pre_cond.add_condition(Predicated(false));
1189 : : } else {
1190 : : /*----- Non-SIMDfied print does not support SIMD. -----*/
1191 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
1192 : : }
1193 : :
1194 : 0 : return pre_cond;
1195 [ # # # # ]: 0 : }
1196 : :
1197 : : template<bool SIMDfied>
1198 : 0 : void Print<SIMDfied>::execute(const Match<Print> &M, setup_t, pipeline_t, teardown_t)
1199 : : {
1200 : 0 : M_insist(bool(M.result_set_factory), "`wasm::Print` must have a factory for the result set");
1201 : :
1202 [ # # # # ]: 0 : auto result_set_schema = M.print_op.schema().drop_constants().deduplicate();
1203 [ # # # # ]: 0 : write_result_set(result_set_schema, *M.result_set_factory, M.result_set_window_size, *M.child);
1204 : 0 : }
1205 : :
1206 : :
1207 : : /*======================================================================================================================
1208 : : * Scan
1209 : : *====================================================================================================================*/
1210 : :
1211 : : template<bool SIMDfied>
1212 : 0 : ConditionSet Scan<SIMDfied>::pre_condition(std::size_t child_idx,
1213 : : const std::tuple<const ScanOperator*> &partial_inner_nodes)
1214 : : {
1215 : 0 : M_insist(child_idx == 0);
1216 : :
1217 : 0 : ConditionSet pre_cond;
1218 : :
1219 : : if constexpr (SIMDfied) {
1220 : 0 : auto &scan = *std::get<0>(partial_inner_nodes);
1221 [ # # # # ]: 0 : auto &table = scan.store().table();
1222 : :
1223 : : /*----- SIMDfied scan needs the data layout to support SIMD. -----*/
1224 [ # # # # : 0 : if (not supports_simd(table.layout(), table.schema(scan.alias()), scan.schema()))
# # # # #
# # # #
# ]
1225 [ # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1226 : :
1227 : : /*----- SIMDfied scan needs the number of rows to load be a whole multiple of the number of SIMD lanes used. -*/
1228 [ # # # # : 0 : if (scan.store().num_rows() % get_num_simd_lanes(table.layout(), table.schema(scan.alias()), scan.schema()) != 0)
# # # # #
# # # # #
# # # # ]
1229 [ # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1230 : : }
1231 : :
1232 : 0 : return pre_cond;
1233 : 0 : }
1234 : :
1235 : : template<bool SIMDfied>
1236 : 0 : ConditionSet Scan<SIMDfied>::post_condition(const Match<Scan> &M)
1237 : : {
1238 : 0 : ConditionSet post_cond;
1239 : :
1240 : : /*----- Scan does not introduce predication. -----*/
1241 [ # # # # : 0 : post_cond.add_condition(Predicated(false));
# # # # ]
1242 : :
1243 : : if constexpr (SIMDfied) {
1244 : : /*----- SIMDfied scan introduces SIMD vectors with respective number of lanes. -----*/
1245 [ # # # # ]: 0 : auto &table = M.scan.store().table();
1246 [ # # # # : 0 : const auto num_simd_lanes = get_num_simd_lanes(table.layout(), table.schema(M.scan.alias()), M.scan.schema());
# # # # #
# # # ]
1247 [ # # # # ]: 0 : post_cond.add_condition(SIMD(num_simd_lanes));
1248 : : } else {
1249 : : /*----- Non-SIMDfied scan does not introduce SIMD. -----*/
1250 [ # # # # ]: 0 : post_cond.add_condition(NoSIMD());
1251 : : }
1252 : :
1253 : : /*----- Check if any attribute of scanned table is assumed to be sorted. -----*/
1254 : 0 : Sortedness::order_t orders;
1255 [ # # # # : 0 : for (auto &e : M.scan.schema()) {
# # # # #
# # # # #
# # ]
1256 : 0 : auto pred = [&e](const auto &p){ return e.id == p.first; };
1257 [ # # # # : 0 : if (auto it = std::find_if(options::sorted_attributes.cbegin(), options::sorted_attributes.cend(), pred);
# # # # #
# # # ]
1258 : 0 : it != options::sorted_attributes.cend())
1259 : : {
1260 [ # # # # : 0 : orders.add(e.id, it->second ? Sortedness::O_ASC : Sortedness::O_DESC);
# # # # ]
1261 : 0 : }
1262 : : }
1263 [ # # # # : 0 : if (not orders.empty())
# # # # ]
1264 [ # # # # : 0 : post_cond.add_condition(Sortedness(std::move(orders)));
# # # # ]
1265 : :
1266 : 0 : return post_cond;
1267 [ # # # # ]: 0 : }
1268 : :
1269 : : template<bool SIMDfied>
1270 : 10 : void Scan<SIMDfied>::execute(const Match<Scan> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
1271 : : {
1272 : 10 : auto &schema = M.scan.schema();
1273 : 10 : auto &table = M.scan.store().table();
1274 : :
1275 [ + - + - : 10 : M_insist(schema == schema.drop_constants().deduplicate(), "schema of `ScanOperator` must not contain NULL or duplicates");
+ - # # #
# # # ]
1276 : 10 : M_insist(not table.layout().is_finite(), "layout for `wasm::Scan` must be infinite");
1277 : :
1278 : 10 : Var<U32x1> tuple_id; // default initialized to 0
1279 : :
1280 : : /*----- Compute possible number of SIMD lanes and decide which to use with regard to other operators preferences. */
1281 [ + - + - : 10 : const auto layout_schema = table.schema(M.scan.alias());
+ - # # #
# # # ]
1282 [ + - + - : 10 : CodeGenContext::Get().update_num_simd_lanes_preferred(options::simd_lanes); // set configured preference
# # # # ]
1283 : 10 : const auto num_simd_lanes_preferred =
1284 [ + - + - : 10 : CodeGenContext::Get().num_simd_lanes_preferred(); // get other operators preferences
# # # # ]
1285 : 10 : const std::size_t num_simd_lanes =
1286 : 0 : SIMDfied ? (options::double_pumping ? 2 : 1)
1287 [ # # # # : 0 : * std::max(num_simd_lanes_preferred, get_num_simd_lanes(table.layout(), layout_schema, schema))
# # ]
1288 : : : 1;
1289 [ + - + - : 10 : CodeGenContext::Get().set_num_simd_lanes(num_simd_lanes);
# # # # ]
1290 : :
1291 : : /*----- Import the number of rows of `table`. -----*/
1292 [ + - + - : 10 : U32x1 num_rows = get_num_rows(table.name());
# # # # ]
1293 : :
1294 : : /*----- If no attributes must be loaded, generate a loop just executing the pipeline `num_rows`-times. -----*/
1295 [ + - - + : 10 : if (schema.num_entries() == 0) {
# # # # ]
1296 [ # # # # ]: 0 : setup();
1297 [ # # # # : 0 : WHILE (tuple_id < num_rows) {
# # # # #
# # # # #
# # ]
1298 [ # # # # ]: 0 : tuple_id += uint32_t(num_simd_lanes);
1299 [ # # # # ]: 0 : pipeline();
1300 : : }
1301 [ # # # # ]: 0 : teardown();
1302 : 0 : return;
1303 : : }
1304 : :
1305 : : /*----- Import the base address of the mapped memory. -----*/
1306 [ + - + - : 10 : Ptr<void> base_address = get_base_address(table.name());
# # # # ]
1307 : :
1308 : : /*----- Emit setup code *before* compiling data layout to not overwrite its temporary boolean variables. -----*/
1309 [ + - # # ]: 10 : setup();
1310 : :
1311 : : /*----- Compile data layout to generate sequential load from table. -----*/
1312 [ + + - + : 10 : static Schema empty_schema;
# # # # ]
1313 [ + - + - : 30 : auto [inits, loads, jumps] = compile_load_sequential(schema, empty_schema, base_address, table.layout(),
+ - # # #
# # # ]
1314 : 0 : num_simd_lanes, layout_schema, tuple_id);
1315 : :
1316 : : /*----- Generate the loop for the actual scan, with the pipeline emitted into the loop body. -----*/
1317 [ + - # # ]: 10 : inits.attach_to_current();
1318 [ + - + - : 20 : WHILE (tuple_id < num_rows) {
+ - + - #
# # # # #
# # ]
1319 [ + - # # ]: 10 : loads.attach_to_current();
1320 [ + - # # ]: 10 : pipeline();
1321 [ + - # # ]: 10 : jumps.attach_to_current();
1322 : : }
1323 : :
1324 : : /*----- Emit teardown code. -----*/
1325 [ + - # # ]: 10 : teardown();
1326 [ - + # # ]: 10 : }
1327 : :
1328 : :
1329 : : /*======================================================================================================================
1330 : : * Index Scan
1331 : : *====================================================================================================================*/
1332 : :
1333 : : template<idx::IndexMethod IndexMethod>
1334 : 0 : ConditionSet IndexScan<IndexMethod>::pre_condition(std::size_t child_idx,
1335 : : const std::tuple<const FilterOperator*,
1336 : : const ScanOperator*> &partial_inner_nodes)
1337 : : {
1338 : 0 : M_insist(child_idx == 0);
1339 : :
1340 : : /*----- Check if index on one of the attributes in filter condition exists. -----*/
1341 : 0 : auto &filter = *std::get<0>(partial_inner_nodes);
1342 : 0 : auto &cnf = filter.filter();
1343 : :
1344 : 0 : M_insist(not cnf.empty(), "Filter condition must not be empty");
1345 : :
1346 : 0 : auto &scan = *std::get<1>(partial_inner_nodes);
1347 : 0 : auto &table = scan.store().table();
1348 : :
1349 : 0 : Catalog &C = Catalog::Get();
1350 : 0 : auto &DB = C.get_database_in_use();
1351 : :
1352 : : /* Extract attributes from filter condition. */
1353 : 0 : std::vector<Schema::Identifier> ids;
1354 [ # # # # : 0 : for (auto &entry : cnf.get_required()) {
# # # # #
# # # # #
# # ]
1355 : 0 : Schema::Identifier &id = entry.id;
1356 [ # # # # : 0 : M_insist(table.name() == id.prefix, "Table name should match designator table name");
# # # # #
# # # # #
# # ]
1357 : :
1358 : : /* Keep attributes for which an index exists. */
1359 [ # # # # : 0 : if (DB.has_index(table.name(), id.name, IndexMethod))
# # # # #
# # # ]
1360 [ # # # # ]: 0 : ids.push_back(id);
1361 : : }
1362 [ # # # # ]: 0 : if (ids.empty()) // no usable index found
1363 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1364 : :
1365 : : /*----- Check if filter condition is supported. -----*/
1366 : : /* We currently only support filter conditions of the following form.
1367 : : * 1. Point: condition with a single equality predicate, e.g.
1368 : : * x = 42.
1369 : : * 2. One-sided range: condition with a single greater/greater-or-equal/less/less-or-equal predicate, e.g.
1370 : : * x > 42.
1371 : : * 3. Two-sided range: condition with a greater/greater-or-equal predicate and a less/less-or-equal predicate, e.g.
1372 : : * x > 42 AND x <= 89.
1373 : : * Attributes may appear on either side. The other side is required to be a `Constant`. */
1374 [ # # # # ]: 0 : if (ids.size() > 1) // conditions with more than one attribute currently not supported
1375 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1376 : :
1377 : 0 : bool has_lo_bound = false;
1378 : 0 : bool has_hi_bound = false;
1379 [ # # # # ]: 0 : for (auto &clause : cnf) {
1380 [ # # # # ]: 0 : if (clause.size() != 1) // disjunctions not supported
1381 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1382 : :
1383 : 0 : auto &predicate = clause[0];
1384 [ # # # # : 0 : if (predicate.negative()) // negated predicates not supported
# # # # ]
1385 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1386 : :
1387 [ # # # # : 0 : auto expr = cast<const BinaryExpr>(&predicate.expr());
# # # # ]
1388 [ # # # # ]: 0 : if (not expr) // not a binary expression
1389 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1390 : :
1391 [ # # # # ]: 0 : bool has_attribute_left = is<const Designator>(expr->lhs);
1392 [ # # # # ]: 0 : auto &attribute = has_attribute_left ? *expr->lhs : *expr->rhs;
1393 [ # # # # ]: 0 : auto &constant = has_attribute_left ? *expr->rhs : *expr->lhs;
1394 [ # # # # : 0 : if (not is<const Designator>(attribute)) // attribute must be on lhs
# # # # ]
1395 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1396 [ # # # # : 0 : if (not is_valid_bound(constant))
# # # # ]
1397 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1398 : :
1399 [ # # # # : 0 : switch(expr->tok.type) {
# # # # ]
1400 : : default:
1401 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1402 : : case TK_EQUAL:
1403 [ # # # # : 0 : if (not has_lo_bound and not has_hi_bound) { // lo and hi bound not yet set
# # # # ]
1404 : 0 : has_lo_bound = has_hi_bound = true;
1405 : 0 : } else {
1406 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1407 : : }
1408 : 0 : break;
1409 : : case TK_GREATER:
1410 : : case TK_GREATER_EQUAL:
1411 [ # # # # : 0 : if (has_attribute_left and not has_lo_bound) { // attribute on lhs, lo bound not yet set
# # # # ]
1412 : 0 : has_lo_bound = true;
1413 [ # # # # : 0 : } else if (not has_attribute_left and not has_hi_bound) { // attribute on rhs, hi bound not yet set
# # # # ]
1414 : 0 : has_hi_bound = true;
1415 : 0 : } else {
1416 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1417 : : }
1418 : 0 : break;
1419 : : case TK_LESS:
1420 : : case TK_LESS_EQUAL:
1421 [ # # # # : 0 : if (has_attribute_left and not has_hi_bound) { // attribute on lhs, hi bound not yet set
# # # # ]
1422 : 0 : has_hi_bound = true;
1423 [ # # # # ]: 0 : } else if (not has_attribute_left and not has_lo_bound) { // attribute on rhs, lo bound not yet set
1424 : 0 : has_lo_bound = true;
1425 : 0 : } else {
1426 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
1427 : : }
1428 : 0 : break;
1429 : : }
1430 : : }
1431 : 0 : return ConditionSet();
1432 : 0 : }
1433 : :
1434 : : template<idx::IndexMethod IndexMethod>
1435 : 0 : ConditionSet IndexScan<IndexMethod>::post_condition(const Match<IndexScan<IndexMethod>> &M)
1436 : : {
1437 : 0 : ConditionSet post_cond;
1438 : :
1439 : : /*----- Index scan does not introduce predication. -----*/
1440 [ # # # # : 0 : post_cond.add_condition(Predicated(false));
# # # # ]
1441 : :
1442 : : /*----- Non-SIMDfied index scan does not introduce SIMD. -----*/
1443 [ # # # # : 0 : post_cond.add_condition(NoSIMD());
# # # # ]
1444 : :
1445 : : /*----- Index scan introduces sortedness on indexed attribute. -----*/
1446 : : /* Extract identifier from cnf. */
1447 [ # # # # ]: 0 : auto &cnf = M.filter.filter();
1448 [ # # # # ]: 0 : Schema designators = cnf.get_required();
1449 [ # # # # : 0 : M_insist(designators.num_entries() == 1, "filter condition must contain exactly one designator");
# # # # ]
1450 [ # # # # ]: 0 : Schema::Identifier &id = designators[0].id;
1451 : :
1452 : : /* Add sortedness post condition. */
1453 : 0 : Sortedness::order_t orders;
1454 [ # # # # : 0 : orders.add(id, Sortedness::O_ASC);
# # # # ]
1455 [ # # # # : 0 : post_cond.add_condition(Sortedness(std::move(orders)));
# # # # ]
1456 : :
1457 : 0 : return post_cond;
1458 [ # # # # ]: 0 : }
1459 : :
1460 : : template<idx::IndexMethod IndexMethod>
1461 : 0 : double IndexScan<IndexMethod>::cost(const Match<IndexScan<IndexMethod>>&)
1462 : : {
1463 : : /* TODO: add meaningful cost function. */
1464 : 0 : return 0.0;
1465 : : }
1466 : :
1467 : : template<idx::IndexMethod IndexMethod, typename Index, sql_type SqlT>
1468 : 0 : void index_scan_codegen_compilation(const Index &index, const index_scan_bounds_t &bounds,
1469 : : const Match<IndexScan<IndexMethod>> &M,
1470 : : setup_t setup, pipeline_t pipeline, teardown_t teardown) {
1471 : : using key_type = Index::key_type;
1472 : : using sql_type = SqlT;
1473 : :
1474 [ # # # # : 0 : if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::CALLBACK) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1475 : : /*----- Resolve callback function names. -----*/
1476 : : const char *scan_fn, *lower_bound_fn, *upper_bound_fn;
1477 : : #define SET_CALLBACK_FNS(INDEX, KEY) \
1478 : : scan_fn = M_STR(idx_scan_##INDEX##_##KEY); \
1479 : : lower_bound_fn = M_STR(idx_lower_bound_##INDEX##_##KEY); \
1480 : : upper_bound_fn = M_STR(idx_upper_bound_##INDEX##_##KEY)
1481 : :
1482 : : #define RESOLVE_KEYTYPE(INDEX) \
1483 : : if constexpr(std::same_as<SqlT, _Boolx1>) { \
1484 : : SET_CALLBACK_FNS(INDEX, b); \
1485 : : } else if constexpr(std::same_as<sql_type, _I8x1>) { \
1486 : : SET_CALLBACK_FNS(INDEX, i1); \
1487 : : } else if constexpr(std::same_as<sql_type, _I16x1>) { \
1488 : : SET_CALLBACK_FNS(INDEX, i2); \
1489 : : } else if constexpr(std::same_as<sql_type, _I32x1>) { \
1490 : : SET_CALLBACK_FNS(INDEX, i4); \
1491 : : } else if constexpr(std::same_as<sql_type, _I64x1>) { \
1492 : : SET_CALLBACK_FNS(INDEX, i8); \
1493 : : } else if constexpr(std::same_as<sql_type, _Floatx1>) { \
1494 : : SET_CALLBACK_FNS(INDEX, f); \
1495 : : } else if constexpr(std::same_as<sql_type, _Doublex1>) { \
1496 : : SET_CALLBACK_FNS(INDEX, d); \
1497 : : } else if constexpr(std::same_as<sql_type, NChar>) { \
1498 : : SET_CALLBACK_FNS(IDNEX, p); \
1499 : : } else { \
1500 : : M_unreachable("incompatible SQL type"); \
1501 : : }
1502 : : if constexpr(is_specialization<Index, idx::ArrayIndex>) {
1503 : 0 : RESOLVE_KEYTYPE(array)
1504 : : } else if constexpr(is_specialization<Index, idx::RecursiveModelIndex>) {
1505 : 0 : RESOLVE_KEYTYPE(rmi)
1506 : : } else {
1507 : : M_unreachable("unknown index type");
1508 : : }
1509 : : #undef RESOLVE_KEYTYPE
1510 : : #undef SET_CALLBACK_FNS
1511 : :
1512 : : /*----- Add index to context. -----*/
1513 : 0 : auto &context = WasmEngine::Get_Wasm_Context_By_ID(Module::ID());
1514 : 0 : U64x1 index_id(context.add_index(index));
1515 : :
1516 : : /*----- Emit host calls to query the index for lo and hi bounds. -----*/
1517 : 0 : auto compile_bound_lookup = [&](const ast::Expr &bound, bool is_lower_bound) {
1518 : 0 : auto [constant, is_negative] = get_valid_bound(bound);
1519 : 0 : auto c = Interpreter::eval(constant);
1520 : : key_type _key;
1521 : : if constexpr(m::boolean<key_type>) {
1522 : 0 : _key = bool(c);
1523 : 0 : M_insist(not is_negative, "boolean cannot be negative");
1524 : : } else if constexpr(m::integral<key_type>) {
1525 : 0 : auto i64 = int64_t(c);
1526 : 0 : M_insist(std::in_range<key_type>(i64), "integeral constant must be in range");
1527 : 0 : _key = key_type(i64);
1528 [ # # # # : 0 : _key = is_negative ? -_key : _key;
# # # # #
# # # # #
# # ]
1529 : : } else if constexpr(std::same_as<float, key_type>) {
1530 : 0 : auto d = double(c);
1531 : 0 : _key = key_type(d);
1532 : 0 : M_insist(_key == d, "downcasting should not impact precision");
1533 [ # # # # ]: 0 : _key = is_negative ? -_key : _key;
1534 : : } else if constexpr(std::same_as<double, key_type>) {
1535 : 0 : _key = double(c);
1536 [ # # # # ]: 0 : _key = is_negative ? -_key : _key;
1537 : : } else if constexpr(std::same_as<const char*, key_type>) {
1538 : 0 : _key = reinterpret_cast<const char*>(c.as_p());
1539 : 0 : M_insist(not is_negative, "string cannot be negative");
1540 : : }
1541 : :
1542 : 0 : std::optional<typename sql_type::primitive_type> key;
1543 [ # # # # : 0 : if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::INLINE) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1544 : : if constexpr (std::same_as<sql_type, NChar>) {
1545 [ # # # # : 0 : key.emplace(CodeGenContext::Get().get_literal_address(_key));
# # ]
1546 : : } else {
1547 [ # # # # : 0 : key.emplace(_key);
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1548 : : }
1549 [ # # # # : 0 : } else if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::MEMORY) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1550 : : /* If we materialize before calling the bound functions, the key parameter is independent of the bounds.
1551 : : * As a result, queries that only differ in the filter predicate are compiled to the exact same
1552 : : * WebAssembly code, enabling caching of compiled plans in V8. */
1553 : : if constexpr (std::same_as<sql_type, NChar>) {
1554 [ # # # # ]: 0 : uint32_t *key_address = Module::Allocator().raw_malloc<uint32_t>();
1555 [ # # # # ]: 0 : *key_address = CodeGenContext::Get().get_literal_raw_address(_key);
1556 : :
1557 [ # # ]: 0 : Ptr<U32x1> key_ptr(key_address);
1558 [ # # # # : 0 : key.emplace(U32x1(*key_ptr).to<char*>(), false, as<const CharacterSequence>(bound.type()));
# # # # #
# # # ]
1559 : 0 : } else {
1560 [ # # # # : 0 : auto *key_address = Module::Allocator().raw_malloc<typename sql_type::type>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1561 : 0 : *key_address = _key;
1562 : :
1563 [ # # # # : 0 : Ptr<typename sql_type::primitive_type> key_ptr(key_address);
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1564 [ # # # # : 0 : key.emplace(*key_ptr);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1565 : 0 : }
1566 : 0 : } else {
1567 [ # # # # : 0 : M_unreachable("unknown materialization strategy");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1568 : : }
1569 [ # # # # : 0 : M_insist(bool(key), "key must be set");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1570 [ # # # # : 0 : return Module::Get().emit_call<uint32_t>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1571 [ # # # # : 0 : /* fn= */ is_lower_bound ? lower_bound_fn : upper_bound_fn,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1572 [ # # # # : 0 : /* index_id= */ index_id.clone(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1573 [ # # # # : 0 : /* key= */ *key
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1574 : : );
1575 : 0 : };
1576 [ # # # # : 0 : Var<U32x1> lo(bool(bounds.lo) ? compile_bound_lookup(bounds.lo->get(), bounds.is_inclusive_lo)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1577 [ # # # # : 0 : : U32x1(0));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1578 [ # # # # : 0 : const Var<U32x1> hi(bool(bounds.hi) ? compile_bound_lookup(bounds.hi->get(), not bounds.is_inclusive_hi)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1579 [ # # # # : 0 : : U32x1(index.num_entries()));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1580 [ # # # # : 0 : Wasm_insist(lo <= hi, "bounds need to be valid");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1581 : :
1582 : : /*----- Allocate memory for communication to host. -----*/
1583 [ # # # # : 0 : M_insist(std::in_range<uint32_t>(M.batch_size), "should fit in uint32_t");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1584 : :
1585 : : /* Determine alloc size as minimum of number of results and command-line parameter batch size, where 0 is
1586 : : * interpreted as infinity. */
1587 [ # # # # : 0 : const Var<U32x1> alloc_size([&](){
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1588 : 0 : U32x1 num_results = hi - lo;
1589 [ # # # # : 0 : U32x1 num_results_cpy = num_results.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1590 [ # # # # : 0 : U32x1 batch_size = M.batch_size == 0 ? num_results.clone() : U32x1(M.batch_size);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1591 [ # # # # : 0 : U32x1 batch_size_cpy = batch_size.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1592 [ # # # # : 0 : return Select(batch_size < num_results, batch_size_cpy, num_results_cpy);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1593 : 0 : }());
1594 [ # # # # : 0 : Ptr<U32x1> buffer_address = Module::Allocator().malloc<uint32_t>(alloc_size);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1595 : :
1596 : : /*----- Emit setup code *after* allocating memory to guarantee sequential memory allocation for pipeline. -----*/
1597 [ # # # # : 0 : setup();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1598 : :
1599 : : /*----- Emit loop code. -----*/
1600 [ # # # # : 0 : Var<U32x1> num_tuples_in_batch;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1601 [ # # # # : 0 : Var<Ptr<U32x1>> ptr;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1602 [ # # # # : 0 : WHILE (lo < hi) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1603 [ # # # # : 0 : num_tuples_in_batch = Select(hi - lo > alloc_size, alloc_size, hi - lo);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1604 : : /* Call host to fill buffer memory with next batch of tuple ids. */
1605 [ # # # # : 0 : Module::Get().emit_call<void>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1606 : 0 : /* fn= */ scan_fn,
1607 [ # # # # : 0 : /* index_id= */ index_id,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1608 [ # # # # : 0 : /* entry_offset= */ lo.val(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1609 [ # # # # : 0 : /* address= */ buffer_address.clone(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1610 [ # # # # : 0 : /* batch_size= */ num_tuples_in_batch.val()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1611 : : );
1612 [ # # # # : 0 : lo += num_tuples_in_batch;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1613 [ # # # # : 0 : ptr = buffer_address.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1614 [ # # # # : 0 : WHILE(num_tuples_in_batch > 0U) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1615 [ # # # # : 0 : static Schema empty_schema;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1616 [ # # # # : 0 : compile_load_point_access(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1617 : 0 : /* tuple_value_schema= */ M.scan.schema(),
1618 : : /* tuple_address_schema= */ empty_schema,
1619 [ # # # # : 0 : /* base_address= */ get_base_address(M.scan.store().table().name()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1620 [ # # # # : 0 : /* layout= */ M.scan.store().table().layout(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1621 [ # # # # : 0 : /* layout_schema= */ M.scan.store().table().schema(M.scan.alias()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1622 [ # # # # : 0 : /* tuple_id= */ *ptr
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1623 : : );
1624 [ # # # # : 0 : pipeline();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1625 [ # # # # : 0 : num_tuples_in_batch -= 1U;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1626 [ # # # # : 0 : ptr += 1;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1627 : : }
1628 : : }
1629 : :
1630 : : /*----- Emit teardown code. -----*/
1631 [ # # # # : 0 : teardown();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1632 : :
1633 : : /*----- Free buffer memory. -----*/
1634 [ # # # # : 0 : IF (alloc_size > U32x1(0)) { // only free if actually allocated
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1635 : 0 : Module::Allocator().free(buffer_address, alloc_size);
1636 : 0 : };
1637 [ # # # # : 0 : } else if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::EXPOSED_MEMORY) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1638 : 0 : M_unreachable("not implemented yet");
1639 : : } else {
1640 : 0 : M_unreachable("unknown compilation stategy");
1641 : : }
1642 : 0 : }
1643 : :
1644 : : template<idx::IndexMethod IndexMethod, typename Index>
1645 : 0 : void index_scan_codegen_interpretation(const Index &index, const index_scan_bounds_t &bounds,
1646 : : const Match<IndexScan<IndexMethod>> &M,
1647 : : setup_t setup, pipeline_t pipeline, teardown_t teardown)
1648 : : {
1649 : : using key_type = Index::key_type;
1650 : :
1651 [ # # # # : 0 : static Schema empty_schema;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1652 : :
1653 : : /*----- Interpret lo and hi bounds to retrieve index scan range -----*/
1654 : 0 : auto interpret_and_lookup_bound = [&](const ast::Expr &bound, bool is_lower_bound) -> std::size_t {
1655 : 0 : auto [constant, is_negative] = get_valid_bound(bound);
1656 : 0 : auto c = Interpreter::eval(constant);
1657 : : key_type key;
1658 : : if constexpr(m::boolean<key_type>) {
1659 : 0 : key = bool(c);
1660 : 0 : M_insist(not is_negative, "boolean cannot be negative");
1661 : : } else if constexpr(m::integral<key_type>) {
1662 : 0 : auto i64 = int64_t(c);
1663 : 0 : M_insist(std::in_range<key_type>(i64), "integeral constant must be in range");
1664 : 0 : key = key_type(i64);
1665 [ # # # # : 0 : key = is_negative ? -key : key;
# # # # #
# # # # #
# # ]
1666 : : } else if constexpr(std::same_as<float, key_type>) {
1667 : 0 : auto d = double(c);
1668 : 0 : key = key_type(d);
1669 : 0 : M_insist(key == d, "downcasting should not impact precision");
1670 [ # # # # ]: 0 : key = is_negative ? -key : key;
1671 : : } else if constexpr(std::same_as<double, key_type>) {
1672 : 0 : key = double(c);
1673 [ # # # # ]: 0 : key = is_negative ? -key : key;
1674 : : } else if constexpr(std::same_as<const char*, key_type>) {
1675 : 0 : key = reinterpret_cast<const char*>(c.as_p());
1676 : 0 : M_insist(not is_negative, "string cannot be negative");
1677 : : }
1678 [ # # # # : 0 : return std::distance(index.begin(), is_lower_bound ? index.lower_bound(key)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1679 : 0 : : index.upper_bound(key));
1680 : : };
1681 [ # # # # : 0 : std::size_t lo = bool(bounds.lo) ? interpret_and_lookup_bound(bounds.lo->get(), bounds.is_inclusive_lo)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1682 : : : 0UL;
1683 [ # # # # : 0 : std::size_t hi = bool(bounds.hi) ? interpret_and_lookup_bound(bounds.hi->get(), not bounds.is_inclusive_hi)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1684 : 0 : : index.num_entries();
1685 : 0 : M_insist(lo <= hi, "bounds need to be valid");
1686 : :
1687 [ # # # # : 0 : if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::MEMORY) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1688 : : /*----- Allocate sufficient memory for results. -----*/
1689 : 0 : uint32_t num_results = hi - lo;
1690 : 0 : uint32_t *buffer_address = Module::Allocator().raw_malloc<uint32_t>(num_results + 1); // +1 for storing number of results itself
1691 : :
1692 : : /*----- Perform index scan and fill memory with number of results and results. -----*/
1693 : 0 : uint32_t *buffer_ptr = buffer_address;
1694 : 0 : *buffer_ptr = num_results; // store in memory to enable caching
1695 : 0 : ++buffer_ptr;
1696 [ # # # # : 0 : for (auto it = index.begin() + lo; it != index.begin() + hi; ++it) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1697 : 0 : M_insist(std::in_range<uint32_t>(it->second), "tuple id must fit in uint32_t");
1698 : 0 : *buffer_ptr = it->second;
1699 : 0 : ++buffer_ptr;
1700 : 0 : }
1701 : :
1702 : : /*----- Emit setup code *after* allocating memory to guarantee sequential memory allocation for pipeline. -----*/
1703 : 0 : setup();
1704 : :
1705 : : /*----- Emit loop code. -----*/
1706 : 0 : Ptr<U32x1> base(buffer_address + 1); // +1 to skip stored number of results
1707 [ # # # # : 0 : Var<Ptr<U32x1>> ptr(base.clone());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1708 [ # # # # : 0 : const Var<Ptr<U32x1>> end(base + U32x1(*Ptr<U32x1>(buffer_address)).make_signed());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1709 [ # # # # : 0 : WHILE(ptr < end) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1710 [ # # # # : 0 : compile_load_point_access(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1711 : 0 : /* tuple_value_schema= */ M.scan.schema(),
1712 : : /* tuple_address_schema= */ empty_schema,
1713 [ # # # # : 0 : /* base_address= */ get_base_address(M.scan.store().table().name()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1714 [ # # # # : 0 : /* layout= */ M.scan.store().table().layout(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1715 [ # # # # : 0 : /* layout_schema= */ M.scan.store().table().schema(M.scan.alias()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1716 [ # # # # : 0 : /* tuple_id= */ *ptr
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1717 : : );
1718 [ # # # # : 0 : pipeline();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1719 [ # # # # : 0 : ptr += 1;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1720 : : }
1721 : :
1722 : : /*----- Emit teardown code. -----*/
1723 [ # # # # : 0 : teardown();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1724 [ # # # # : 0 : } else if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::INLINE) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1725 : : /*----- Define function that emits code for loading and executing pipeline for a single tuple. -----*/
1726 [ # # # # : 0 : FUNCTION(index_scan_parent_pipeline, void(uint32_t))
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1727 : : {
1728 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1729 : :
1730 : : /*----- Emit setup code. -----*/
1731 [ # # # # : 0 : setup();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1732 : :
1733 : : /*----- Load tuple. ----- */
1734 [ # # # # : 0 : compile_load_point_access(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1735 : 0 : /* tuple_value_schema= */ M.scan.schema(),
1736 : : /* tuple_address_schema= */ empty_schema,
1737 [ # # # # : 0 : /* base_address= */ get_base_address(M.scan.store().table().name()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1738 [ # # # # : 0 : /* layout= */ M.scan.store().table().layout(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1739 [ # # # # : 0 : /* layout_schema= */ M.scan.store().table().schema(M.scan.alias()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1740 [ # # # # : 0 : /* tuple_id= */ PARAMETER(0)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1741 : : );
1742 : :
1743 : : /*----- Emit pipeline code. -----*/
1744 [ # # # # : 0 : pipeline();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1745 : :
1746 : : /*----- Emit teardown code. -----*/
1747 [ # # # # : 0 : teardown();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1748 : 0 : }
1749 : :
1750 : : /*----- Perform index sequential scan, emit code to execute pipeline for each tuple. -----*/
1751 [ # # # # : 0 : for (auto it = index.begin() + lo; it != index.begin() + hi; ++it) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1752 [ # # # # : 0 : M_insist(std::in_range<uint32_t>(it->second), "tuple id must fit in uint32_t");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1753 [ # # # # : 0 : index_scan_parent_pipeline(uint32_t(it->second));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1754 : 0 : }
1755 : 0 : } else {
1756 : 0 : M_unreachable("unknown materialization strategy");
1757 : : }
1758 : 0 : }
1759 : :
1760 : : template<idx::IndexMethod IndexMethod, typename Index, sql_type SqlT>
1761 : 0 : void index_scan_codegen_hybrid(const Index &index, const index_scan_bounds_t &bounds,
1762 : : const Match<IndexScan<IndexMethod>> &M,
1763 : : setup_t setup, pipeline_t pipeline, teardown_t teardown)
1764 : : {
1765 : : using key_type = Index::key_type;
1766 : : using sql_type = SqlT;
1767 : :
1768 : : /*----- Resolve callback function name. -----*/
1769 : : const char *scan_fn;
1770 : : #define SET_CALLBACK_FNS(INDEX, KEY) \
1771 : : scan_fn = M_STR(idx_scan_##INDEX##_##KEY)
1772 : :
1773 : : #define RESOLVE_KEYTYPE(INDEX) \
1774 : : if constexpr(std::same_as<SqlT, _Boolx1>) { \
1775 : : SET_CALLBACK_FNS(INDEX, b); \
1776 : : } else if constexpr(std::same_as<sql_type, _I8x1>) { \
1777 : : SET_CALLBACK_FNS(INDEX, i1); \
1778 : : } else if constexpr(std::same_as<sql_type, _I16x1>) { \
1779 : : SET_CALLBACK_FNS(INDEX, i2); \
1780 : : } else if constexpr(std::same_as<sql_type, _I32x1>) { \
1781 : : SET_CALLBACK_FNS(INDEX, i4); \
1782 : : } else if constexpr(std::same_as<sql_type, _I64x1>) { \
1783 : : SET_CALLBACK_FNS(INDEX, i8); \
1784 : : } else if constexpr(std::same_as<sql_type, _Floatx1>) { \
1785 : : SET_CALLBACK_FNS(INDEX, f); \
1786 : : } else if constexpr(std::same_as<sql_type, _Doublex1>) { \
1787 : : SET_CALLBACK_FNS(INDEX, d); \
1788 : : } else if constexpr(std::same_as<sql_type, NChar>) { \
1789 : : SET_CALLBACK_FNS(IDNEX, p); \
1790 : : } else { \
1791 : : M_unreachable("incompatible SQL type"); \
1792 : : }
1793 : : if constexpr(is_specialization<Index, idx::ArrayIndex>) {
1794 : 0 : RESOLVE_KEYTYPE(array)
1795 : : } else if constexpr(is_specialization<Index, idx::RecursiveModelIndex>) {
1796 : 0 : RESOLVE_KEYTYPE(rmi)
1797 : : } else {
1798 : : M_unreachable("unknown index type");
1799 : : }
1800 : : #undef RESOLVE_KEYTYPE
1801 : : #undef SET_CALLBACK_FNS
1802 : :
1803 : : /*----- Add index to context. -----*/
1804 : 0 : auto &context = WasmEngine::Get_Wasm_Context_By_ID(Module::ID());
1805 : 0 : U64x1 index_id(context.add_index(index)); ///< id of index in context
1806 : :
1807 : : /*----- Interpret lo and hi bounds to retrieve index scan range -----*/
1808 : 0 : auto interpret_and_lookup_bound = [&](const ast::Expr &bound, bool is_lower_bound) -> std::size_t {
1809 : 0 : auto [constant, is_negative] = get_valid_bound(bound);
1810 : 0 : auto c = Interpreter::eval(constant);
1811 : : key_type key;
1812 : : if constexpr(m::boolean<key_type>) {
1813 : 0 : key = bool(c);
1814 : 0 : M_insist(not is_negative, "boolean cannot be negative");
1815 : : } else if constexpr(m::integral<key_type>) {
1816 : 0 : auto i64 = int64_t(c);
1817 : 0 : M_insist(std::in_range<key_type>(i64), "integeral constant must be in range");
1818 : 0 : key = key_type(i64);
1819 [ # # # # : 0 : key = is_negative ? -key : key;
# # # # #
# # # # #
# # ]
1820 : : } else if constexpr(std::same_as<float, key_type>) {
1821 : 0 : auto d = double(c);
1822 : 0 : key = key_type(d);
1823 : 0 : M_insist(key == d, "downcasting should not impact precision");
1824 [ # # # # ]: 0 : key = is_negative ? -key : key;
1825 : : } else if constexpr(std::same_as<double, key_type>) {
1826 : 0 : key = double(c);
1827 [ # # # # ]: 0 : key = is_negative ? -key : key;
1828 : : } else if constexpr(std::same_as<const char*, key_type>) {
1829 : 0 : key = reinterpret_cast<const char*>(c.as_p());
1830 : 0 : M_insist(not is_negative, "string cannot be negative");
1831 : : }
1832 [ # # # # : 0 : return std::distance(index.begin(), is_lower_bound ? index.lower_bound(key)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1833 : 0 : : index.upper_bound(key));
1834 : : };
1835 [ # # # # : 0 : std::size_t lo = bool(bounds.lo) ? interpret_and_lookup_bound(bounds.lo->get(), bounds.is_inclusive_lo)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1836 : : : 0UL;
1837 [ # # # # : 0 : std::size_t hi = bool(bounds.hi) ? interpret_and_lookup_bound(bounds.hi->get(), not bounds.is_inclusive_hi)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1838 [ # # # # : 0 : : index.num_entries();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1839 [ # # # # : 0 : M_insist(lo <= hi, "bounds need to be valid");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1840 [ # # # # : 0 : M_insist(std::in_range<uint32_t>(lo), "should fit in uint32_t");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1841 [ # # # # : 0 : M_insist(std::in_range<uint32_t>(hi), "should fit in uint32_t");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1842 : :
1843 : : /*----- Materialize offsets hi and lo. -----*/
1844 [ # # # # : 0 : Var<U32x1> begin;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1845 : 0 : std::optional<U32x1> end;
1846 [ # # # # : 0 : if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::INLINE) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1847 [ # # # # : 0 : begin = U32x1(lo);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1848 [ # # # # : 0 : end.emplace(hi);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1849 [ # # # # : 0 : } else if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::MEMORY) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1850 : : /* If we materialize before allocating buffer memory, the addresses are independent of the buffer size. As a
1851 : : * result, queries that only differ in the filter predicate are compiled to the exact same WebAssembly code,
1852 : : * enabling caching of compiled plans in V8. */
1853 [ # # # # : 0 : uint32_t *offset_address = Module::Allocator().raw_malloc<uint32_t>(2);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1854 : 0 : offset_address[0] = uint32_t(lo);
1855 : 0 : offset_address[1] = uint32_t(hi);
1856 : :
1857 [ # # # # : 0 : Ptr<U32x1> offset_ptr(offset_address);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1858 [ # # # # : 0 : begin = *offset_ptr.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1859 [ # # # # : 0 : end.emplace(*(offset_ptr + 1));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1860 : 0 : } else {
1861 [ # # # # : 0 : M_unreachable("unknown materialization strategy");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1862 : : }
1863 [ # # # # : 0 : M_insist(bool(end), "end must be set");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1864 : :
1865 [ # # # # : 0 : if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::CALLBACK) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1866 : : /*----- Allocate buffer memory for communication to host. -----*/
1867 [ # # # # : 0 : M_insist(std::in_range<uint32_t>(M.batch_size), "should fit in uint32_t");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1868 : :
1869 : : /* Determine alloc size as minimum of number of results and command-line parameter batch size, where 0 is
1870 : : * interpreted as infinity. */
1871 [ # # # # : 0 : const Var<U32x1> alloc_size([&](){
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1872 [ # # # # : 0 : U32x1 num_results = end->clone() - begin;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1873 [ # # # # : 0 : U32x1 num_results_cpy = num_results.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1874 [ # # # # : 0 : U32x1 batch_size = M.batch_size == 0 ? num_results.clone() : U32x1(M.batch_size);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1875 [ # # # # : 0 : U32x1 batch_size_cpy = batch_size.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1876 [ # # # # : 0 : return Select(batch_size < num_results, batch_size_cpy, num_results_cpy);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1877 : 0 : }());
1878 [ # # # # : 0 : Ptr<U32x1> buffer_address = Module::Allocator().malloc<uint32_t>(alloc_size);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1879 : :
1880 : : /*----- Emit setup code *after* allocating memory to guarantee sequential memory allocation for pipeline. -----*/
1881 [ # # # # : 0 : setup();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1882 : :
1883 : : /*----- Emit loop code. -----*/
1884 [ # # # # : 0 : Var<U32x1> num_tuples_in_batch;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1885 [ # # # # : 0 : Var<Ptr<U32x1>> ptr;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1886 [ # # # # : 0 : WHILE (begin < end->clone()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1887 [ # # # # : 0 : auto end_cpy = end->clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1888 [ # # # # : 0 : num_tuples_in_batch = Select(*end - begin > alloc_size, alloc_size, end_cpy - begin);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1889 : : /* Call host to fill buffer memory with next batch of tuple ids. */
1890 [ # # # # : 0 : Module::Get().emit_call<void>(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1891 : 0 : /* fn= */ scan_fn,
1892 [ # # # # : 0 : /* index_id= */ index_id,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1893 [ # # # # : 0 : /* entry_offset= */ begin.val(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1894 [ # # # # : 0 : /* address= */ buffer_address.clone(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1895 [ # # # # : 0 : /* batch_size= */ num_tuples_in_batch.val()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1896 : : );
1897 [ # # # # : 0 : begin += num_tuples_in_batch;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1898 [ # # # # : 0 : ptr = buffer_address.clone();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1899 [ # # # # : 0 : WHILE(num_tuples_in_batch > 0U) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1900 [ # # # # : 0 : static Schema empty_schema;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1901 [ # # # # : 0 : compile_load_point_access(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1902 : 0 : /* tuple_value_schema= */ M.scan.schema(),
1903 : : /* tuple_address_schema= */ empty_schema,
1904 [ # # # # : 0 : /* base_address= */ get_base_address(M.scan.store().table().name()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1905 [ # # # # : 0 : /* layout= */ M.scan.store().table().layout(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1906 [ # # # # : 0 : /* layout_schema= */ M.scan.store().table().schema(M.scan.alias()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1907 [ # # # # : 0 : /* tuple_id= */ *ptr
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1908 : : );
1909 [ # # # # : 0 : pipeline();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1910 [ # # # # : 0 : num_tuples_in_batch -= 1U;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1911 [ # # # # : 0 : ptr += 1;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1912 : : }
1913 : 0 : }
1914 : :
1915 : : /*----- Emit teardown code. -----*/
1916 [ # # # # : 0 : teardown();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1917 : :
1918 : : /*----- Free buffer memory. -----*/
1919 [ # # # # : 0 : IF (alloc_size > U32x1(0)) { // only free if actually allocated
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1920 : 0 : Module::Allocator().free(buffer_address, alloc_size);
1921 : 0 : };
1922 [ # # # # : 0 : } else if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::EXPOSED_MEMORY) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1923 [ # # # # : 0 : M_unreachable("not implemented yet");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1924 : : } else {
1925 [ # # # # : 0 : M_unreachable("unknwon compilation strategy");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1926 : : }
1927 : 0 : }
1928 : :
1929 : : /** Resolves the index scan strategy and calls the appropriate codegen function. */
1930 : : template<idx::IndexMethod IndexMethod, typename Index, sql_type SqlT>
1931 : 0 : void index_scan_resolve_strategy(const Index &index, const index_scan_bounds_t &bounds, const Match<IndexScan<IndexMethod>> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
1932 : : {
1933 [ # # # # : 0 : if (options::index_scan_strategy == option_configs::IndexScanStrategy::COMPILATION) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1934 [ # # # # : 0 : index_scan_codegen_compilation<IndexMethod, Index, SqlT>(index, bounds, M, std::move(setup), std::move(pipeline), std::move(teardown));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1935 [ # # # # : 0 : } else if (options::index_scan_strategy == option_configs::IndexScanStrategy::INTERPRETATION) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1936 [ # # # # : 0 : index_scan_codegen_interpretation<IndexMethod, Index>(index, bounds, M, std::move(setup), std::move(pipeline), std::move(teardown));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1937 [ # # # # : 0 : } else if (options::index_scan_strategy == option_configs::IndexScanStrategy::HYBRID) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1938 [ # # # # : 0 : index_scan_codegen_hybrid<IndexMethod, Index, SqlT>(index, bounds, M, std::move(setup), std::move(pipeline), std::move(teardown));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1939 : 0 : } else {
1940 : 0 : M_unreachable("invalid index access strategy");
1941 : : }
1942 : 0 : }
1943 : :
1944 : : /** Resolves the index method and calls the appropriate codegen function. */
1945 : : template<idx::IndexMethod IndexMethod, typename AttrT, sql_type SqlT>
1946 : 0 : void index_scan_resolve_index_method(const index_scan_bounds_t &bounds, const Match<IndexScan<IndexMethod>> &M,
1947 : : setup_t setup, pipeline_t pipeline, teardown_t teardown)
1948 : : {
1949 : : /*----- Lookup index. -----*/
1950 : 0 : Catalog &C = Catalog::Get();
1951 : 0 : auto &DB = C.get_database_in_use();
1952 : 0 : auto &table_name = M.scan.store().table().name();
1953 : 0 : auto &attribute_name = bounds.attribute.id.name;
1954 : 0 : auto &index_base = DB.get_index(table_name, attribute_name, IndexMethod);
1955 : :
1956 : : /*----- Resolve index type. -----*/
1957 : : if constexpr(IndexMethod == idx::IndexMethod::Array and requires { typename idx::ArrayIndex<AttrT>; }) {
1958 : 0 : auto &index = as<const idx::ArrayIndex<AttrT>>(index_base);
1959 [ # # # # : 0 : index_scan_resolve_strategy<IndexMethod, const idx::ArrayIndex<AttrT>, SqlT>(
# # # # #
# # # # #
# # ]
1960 : 0 : index, bounds, M, std::move(setup), std::move(pipeline), std::move(teardown)
1961 : : );
1962 : : } else if constexpr(IndexMethod == idx::IndexMethod::Rmi and requires { typename idx::RecursiveModelIndex<AttrT>; }) {
1963 : 0 : auto &index = as<const idx::RecursiveModelIndex<AttrT>>(index_base);
1964 [ # # # # : 0 : index_scan_resolve_strategy<IndexMethod, const idx::RecursiveModelIndex<AttrT>, SqlT>(
# # # # #
# # # ]
1965 : 0 : index, bounds, M, std::move(setup), std::move(pipeline), std::move(teardown)
1966 : : );
1967 : : } else {
1968 : 0 : M_unreachable("invalid index method");
1969 : : }
1970 : 0 : }
1971 : :
1972 : : /** Resolves the attribute type and calls the appropriate codegen function. */
1973 : : template<idx::IndexMethod IndexMethod>
1974 : 0 : void index_scan_resolve_attribute_type(const Match<IndexScan<IndexMethod>> &M,
1975 : : setup_t setup, pipeline_t pipeline, teardown_t teardown)
1976 : : {
1977 : : /*----- Extract bounds from CNF. -----*/
1978 : 0 : auto &cnf = M.filter.filter();
1979 : 0 : auto bounds = extract_index_scan_bounds(cnf);
1980 : :
1981 : : /*----- Resolve attribute type. -----*/
1982 : : #define RESOLVE_INDEX_METHOD(ATTRTYPE, SQLTYPE) \
1983 : : index_scan_resolve_index_method<IndexMethod, ATTRTYPE, SQLTYPE>(bounds, M, std::move(setup), std::move(pipeline), std::move(teardown))
1984 : :
1985 [ # # # # : 0 : visit(overloaded {
# # # # #
# # # # #
# # # # #
# ]
1986 [ # # # # ]: 0 : [&](const Boolean&) { RESOLVE_INDEX_METHOD(bool, _Boolx1); },
1987 : 0 : [&](const Numeric &n) {
1988 [ # # # # : 0 : switch (n.kind) {
# # ]
1989 : : case Numeric::N_Int:
1990 : : case Numeric::N_Decimal:
1991 [ # # # # : 0 : switch (n.size()) {
# # # # #
# ]
1992 : 0 : default: M_unreachable("invalid size");
1993 [ # # # # ]: 0 : case 8: RESOLVE_INDEX_METHOD(int8_t, _I8x1); break;
1994 [ # # # # ]: 0 : case 16: RESOLVE_INDEX_METHOD(int16_t, _I16x1); break;
1995 [ # # # # ]: 0 : case 32: RESOLVE_INDEX_METHOD(int32_t, _I32x1); break;
1996 [ # # # # ]: 0 : case 64: RESOLVE_INDEX_METHOD(int64_t, _I64x1); break;
1997 : : }
1998 : 0 : break;
1999 : : case Numeric::N_Float:
2000 [ # # # # : 0 : switch (n.size()) {
# # ]
2001 : 0 : default: M_unreachable("invalid size");
2002 [ # # # # ]: 0 : case 32: RESOLVE_INDEX_METHOD(float, _Floatx1); break;
2003 [ # # # # ]: 0 : case 64: RESOLVE_INDEX_METHOD(double, _Doublex1); break;
2004 : : }
2005 : 0 : break;
2006 : : }
2007 : 0 : },
2008 [ # # # # ]: 0 : [&](const CharacterSequence&) { RESOLVE_INDEX_METHOD(const char*, NChar); },
2009 [ # # # # ]: 0 : [&](const Date&) { RESOLVE_INDEX_METHOD(int32_t, _I32x1); },
2010 [ # # # # ]: 0 : [&](const DateTime&) { RESOLVE_INDEX_METHOD(int64_t, _I64x1); },
2011 : 0 : [](auto&&) { M_unreachable("invalid type"); },
2012 : 0 : }, *bounds.attribute.type);
2013 : :
2014 : : #undef RESOLVE_INDEX_METHOD
2015 : 0 : }
2016 : :
2017 : : template<idx::IndexMethod IndexMethod>
2018 : 0 : void IndexScan<IndexMethod>::execute(const Match<IndexScan<IndexMethod>> &M,
2019 : : setup_t setup, pipeline_t pipeline, teardown_t teardown)
2020 : : {
2021 : 0 : auto &schema = M.scan.schema();
2022 [ # # # # : 0 : M_insist(schema == schema.drop_constants().deduplicate(), "Schema of `ScanOperator` must neither contain NULL nor duplicates");
# # # # #
# # # ]
2023 : :
2024 : 0 : auto &table = M.scan.store().table();
2025 : 0 : M_insist(not table.layout().is_finite(), "layout for `wasm::IndexScan` must be infinite");
2026 : :
2027 : : /** Generating code for an index scan requires resolving the following:
2028 : : * 1. attribute type of the indexed attribute,
2029 : : * 2. index method of the index used by the operator,
2030 : : * 3. strategy used for performing the index scan.
2031 : : * We avoid code duplication by handling each step in a separate function and forwarding the result via template
2032 : : * arguments to the next function. */
2033 [ # # # # ]: 0 : index_scan_resolve_attribute_type(M, std::move(setup), std::move(pipeline), std::move(teardown));
2034 : 0 : }
2035 : :
2036 : :
2037 : : /*======================================================================================================================
2038 : : * Filter
2039 : : *====================================================================================================================*/
2040 : :
2041 : : template<bool Predicated>
2042 : 0 : ConditionSet Filter<Predicated>::pre_condition(std::size_t child_idx, const std::tuple<const FilterOperator*>&)
2043 : : {
2044 : 0 : M_insist(child_idx == 0);
2045 : :
2046 : 0 : ConditionSet pre_cond;
2047 : :
2048 : : if constexpr (not Predicated) {
2049 : : /*----- Branching filter does not support SIMD. -----*/
2050 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
2051 : : }
2052 : :
2053 : 0 : return pre_cond;
2054 [ # # # # ]: 0 : }
2055 : :
2056 : : template<bool Predicated>
2057 : 0 : ConditionSet Filter<Predicated>::adapt_post_condition(const Match<Filter>&, const ConditionSet &post_cond_child)
2058 : : {
2059 : 0 : ConditionSet post_cond(post_cond_child);
2060 : :
2061 : : if constexpr (Predicated) {
2062 : : /*----- Predicated filter introduces predication. -----*/
2063 [ # # # # ]: 0 : post_cond.add_or_replace_condition(m::Predicated(true));
2064 : : }
2065 : :
2066 : 0 : return post_cond;
2067 [ # # # # ]: 0 : }
2068 : :
2069 : : template<bool Predicated>
2070 : 0 : double Filter<Predicated>::cost(const Match<Filter> &M)
2071 : : {
2072 : 0 : const cnf::CNF &cond = M.filter.filter();
2073 : 0 : const unsigned cost = std::accumulate(cond.cbegin(), cond.cend(), 0U, [](unsigned cost, const cnf::Clause &clause) {
2074 : 0 : return cost + clause.size();
2075 : : });
2076 : 0 : return cost;
2077 : : }
2078 : :
2079 : : template<bool Predicated>
2080 : 0 : void Filter<Predicated>::execute(const Match<Filter> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
2081 : : {
2082 : : /*----- Set minimal number of SIMD lanes preferred to get fully utilized SIMD vectors for the filter condition. --*/
2083 : 0 : CodeGenContext::Get().update_num_simd_lanes_preferred(16); // set own preference
2084 : :
2085 : : /*----- Execute filter. -----*/
2086 [ # # # # ]: 0 : M.child->execute(
2087 : 0 : /* setup= */ std::move(setup),
2088 [ # # # # ]: 0 : /* pipeline= */ [&, pipeline=std::move(pipeline)](){
2089 : : if constexpr (Predicated) {
2090 : 0 : CodeGenContext::Get().env().add_predicate(M.filter.filter());
2091 : 0 : pipeline();
2092 : : } else {
2093 : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
2094 [ # # # # ]: 0 : IF (CodeGenContext::Get().env().compile<_Boolx1>(M.filter.filter()).is_true_and_not_null()) {
2095 : 0 : pipeline();
2096 : 0 : };
2097 : : }
2098 : 0 : },
2099 : 0 : /* teardown= */ std::move(teardown)
2100 : : );
2101 : 0 : }
2102 : :
2103 : :
2104 : : /*======================================================================================================================
2105 : : * LazyDisjunctiveFilter
2106 : : *====================================================================================================================*/
2107 : :
2108 : 0 : ConditionSet LazyDisjunctiveFilter::pre_condition(std::size_t child_idx, const std::tuple<const FilterOperator*>&)
2109 : : {
2110 : 0 : M_insist(child_idx == 0);
2111 : :
2112 : 0 : ConditionSet pre_cond;
2113 : :
2114 : : /*----- Lazy disjunctive filter does not support SIMD. -----*/
2115 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
2116 : :
2117 : 0 : return pre_cond;
2118 [ # # ]: 0 : }
2119 : :
2120 : 0 : double LazyDisjunctiveFilter::cost(const Match<LazyDisjunctiveFilter> &M)
2121 : : {
2122 : 0 : const cnf::CNF &cond = M.filter.filter();
2123 : 0 : M_insist(cond.size() == 1, "disjunctive filter condition must be a single clause");
2124 : 0 : return cond[0].size() / 2.0; // on avg. half the number of predicates in the clause XXX consider selectivities
2125 : : }
2126 : :
2127 : 0 : void LazyDisjunctiveFilter::execute(const Match<LazyDisjunctiveFilter> &M, setup_t setup, pipeline_t pipeline,
2128 : : teardown_t teardown)
2129 : : {
2130 : 0 : const cnf::Clause &clause = M.filter.filter()[0];
2131 : :
2132 [ # # ]: 0 : M.child->execute(
2133 : 0 : /* setup= */ std::move(setup),
2134 [ # # ]: 0 : /* pipeline= */ [&, pipeline=std::move(pipeline)](){
2135 : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
2136 [ # # ]: 0 : BLOCK(lazy_disjunctive_filter)
2137 : : {
2138 [ # # # # ]: 0 : BLOCK(lazy_disjunctive_filter_then)
2139 : : {
2140 [ # # ]: 0 : for (const cnf::Predicate &pred : clause) {
2141 [ # # # # : 0 : auto cond = CodeGenContext::Get().env().compile<_Boolx1>(*pred);
# # # # ]
2142 [ # # ]: 0 : if (pred.negative())
2143 [ # # # # ]: 0 : GOTO(cond.is_false_and_not_null(), lazy_disjunctive_filter_then); // break to remainder of pipline
2144 : : else
2145 [ # # # # ]: 0 : GOTO(cond.is_true_and_not_null(), lazy_disjunctive_filter_then); // break to remainder of pipline
2146 : 0 : }
2147 [ # # ]: 0 : GOTO(lazy_disjunctive_filter); // skip pipeline
2148 : : }
2149 [ # # ]: 0 : pipeline();
2150 : : }
2151 : 0 : },
2152 : 0 : /* teardown= */ std::move(teardown)
2153 : : );
2154 : 0 : }
2155 : :
2156 : :
2157 : : /*======================================================================================================================
2158 : : * Projection
2159 : : *====================================================================================================================*/
2160 : :
2161 : 0 : ConditionSet Projection::pre_condition(
2162 : : std::size_t child_idx,
2163 : : const std::tuple<const ProjectionOperator*> &partial_inner_nodes)
2164 : : {
2165 : 0 : M_insist(child_idx == 0);
2166 : :
2167 : 0 : ConditionSet pre_cond;
2168 : :
2169 : 0 : auto &projection = *std::get<0>(partial_inner_nodes);
2170 : :
2171 [ # # # # ]: 0 : if (not projection.children().empty()) { // projections starting a pipeline produce only a single tuple, i.e. no SIMD
2172 : : /*----- Projection does only support SIMD if all expressions can be computed using SIMD instructions. -----*/
2173 : 0 : auto is_simd_computable = [](const ast::Expr &e){
2174 : 0 : bool simd_computable = true;
2175 : 0 : visit(overloaded {
2176 : 0 : [&](const ast::BinaryExpr &b) -> void {
2177 [ # # ]: 0 : if (b.lhs->type()->is_character_sequence() or b.rhs->type()->is_character_sequence()) {
2178 : 0 : simd_computable = false; // string operations are not SIMDfiable
2179 : 0 : throw visit_stop_recursion(); // abort recursion
2180 : : }
2181 [ # # # # : 0 : if (b.common_operand_type->is_integral() and b.op().type == TK_SLASH) {
# # ]
2182 : 0 : simd_computable = false; // integer division is not SIMDfiable
2183 : 0 : throw visit_stop_recursion(); // abort recursion
2184 : : }
2185 [ # # ]: 0 : if (b.op().type == TK_PERCENT) {
2186 : 0 : simd_computable = false; // modulo is not SIMDfiable
2187 : 0 : throw visit_stop_recursion(); // abort recursion
2188 : : }
2189 : 0 : },
2190 : 0 : [](auto&) -> void {
2191 : : /* designators, constants, unary expressions, NULL(), INT(), already computed aggregates and results
2192 : : * of a nested query are SIMDfiable; nothing to be done */
2193 : 0 : },
2194 : 0 : }, e, m::tag<m::ast::ConstPreOrderExprVisitor>());
2195 : 0 : return simd_computable;
2196 : : };
2197 : 0 : auto pred = [&](auto &p){ return not is_simd_computable(p.first); };
2198 [ # # # # : 0 : if (std::any_of(projection.projections().cbegin(), projection.projections().cend(), pred))
# # # # ]
2199 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
2200 : 0 : }
2201 : :
2202 : 0 : return pre_cond;
2203 [ # # ]: 0 : }
2204 : :
2205 : 0 : ConditionSet Projection::adapt_post_condition(const Match<Projection> &M, const ConditionSet &post_cond_child)
2206 : : {
2207 : 0 : ConditionSet post_cond(post_cond_child);
2208 : :
2209 : : /*----- Project and rename in duplicated post condition. -----*/
2210 [ # # # # : 0 : M_insist(M.projection.projections().size() == M.projection.schema().num_entries(),
# # # # ]
2211 : : "projections must match the operator's schema");
2212 : 0 : std::vector<std::pair<Schema::Identifier, Schema::Identifier>> old2new;
2213 [ # # ]: 0 : auto p = M.projection.projections().begin();
2214 [ # # # # : 0 : for (auto &e: M.projection.schema()) {
# # # # ]
2215 : 0 : auto pred = [&e](const auto &p) { return p.second == e.id; };
2216 [ # # # # ]: 0 : if (std::find_if(old2new.cbegin(), old2new.cend(), pred) == old2new.cend()) {
2217 [ # # # # ]: 0 : M_insist(p != M.projection.projections().end());
2218 [ # # # # ]: 0 : old2new.emplace_back(Schema::Identifier(p->first.get()), e.id);
2219 : 0 : }
2220 : 0 : ++p;
2221 : : }
2222 [ # # ]: 0 : post_cond.project_and_rename(old2new);
2223 : :
2224 [ # # ]: 0 : if (not M.child) {
2225 : : /*----- Leaf projection does not introduce predication. -----*/
2226 [ # # # # ]: 0 : post_cond.add_condition(Predicated(false));
2227 : :
2228 : : /*----- Leaf projection does not introduce SIMD. -----*/
2229 [ # # # # ]: 0 : post_cond.add_condition(NoSIMD());
2230 : 0 : }
2231 : :
2232 : 0 : return post_cond;
2233 [ # # ]: 0 : }
2234 : :
2235 : 0 : void Projection::execute(const Match<Projection> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
2236 : : {
2237 : 0 : auto execute_projection = [&, pipeline=std::move(pipeline)](){
2238 : 0 : auto &old_env = CodeGenContext::Get().env();
2239 : 0 : Environment new_env; // fresh environment
2240 : :
2241 : : /*----- If predication is used, move predicate to newly created environment. -----*/
2242 [ # # ]: 0 : if (old_env.predicated())
2243 [ # # # # ]: 0 : new_env.add_predicate(old_env.extract_predicate());
2244 : :
2245 : : /*----- Add projections to newly created environment. -----*/
2246 [ # # ]: 0 : M_insist(M.projection.projections().size() == M.projection.schema().num_entries(),
2247 : : "projections must match the operator's schema");
2248 : 0 : auto p = M.projection.projections().begin();
2249 [ # # ]: 0 : for (auto &e: M.projection.schema()) {
2250 [ # # # # : 0 : if (not new_env.has(e.id) and not e.id.is_constant()) { // no duplicate and no constant
# # # # ]
2251 [ # # # # ]: 0 : if (old_env.has(e.id)) {
2252 : : /*----- Migrate compiled expression to new context. ------*/
2253 [ # # # # : 0 : new_env.add(e.id, old_env.get(e.id)); // to retain `e.id` for later compilation of expressions
# # ]
2254 : 0 : } else {
2255 : : /*----- Compile expression. -----*/
2256 [ # # ]: 0 : M_insist(p != M.projection.projections().end());
2257 [ # # ]: 0 : std::visit(overloaded {
2258 : 0 : [&]<typename T, std::size_t L>(Expr<T, L> value) -> void {
2259 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2260 : 0 : Var<Expr<T, L>> var(value); // introduce variable s.t. uses only load from it
2261 [ # # # # : 0 : new_env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2262 : 0 : } else {
2263 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
2264 [ # # # # : 0 : Var<PrimitiveExpr<T, L>> var(value.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2265 [ # # # # : 0 : new_env.add(e.id, Expr<T, L>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2266 : 0 : }
2267 : 0 : },
2268 : 0 : [&](NChar value) -> void {
2269 [ # # ]: 0 : Var<Ptr<Charx1>> var(value.val()); // introduce variable s.t. uses only load from it
2270 [ # # # # : 0 : new_env.add(e.id, NChar(var, value.can_be_null(), value.length(),
# # # # #
# # # ]
2271 [ # # ]: 0 : value.guarantees_terminating_nul()));
2272 : 0 : },
2273 : 0 : [](std::monostate) -> void { M_unreachable("invalid expression"); },
2274 [ # # ]: 0 : }, old_env.compile(p->first));
2275 : : }
2276 : 0 : }
2277 : 0 : ++p;
2278 : : }
2279 : :
2280 : : /*----- Resume pipeline with newly created environment. -----*/
2281 : : {
2282 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(std::move(new_env));
# # ]
2283 [ # # ]: 0 : pipeline();
2284 : 0 : }
2285 : 0 : };
2286 : :
2287 [ # # ]: 0 : if (M.child) {
2288 : : /*----- Set minimal number of SIMD lanes preferred to get fully utilized SIMD vectors *after* the projection. */
2289 : 0 : uint64_t min_size_in_bytes = 16;
2290 [ # # # # ]: 0 : for (auto &p : M.projection.projections()) {
2291 [ # # # # ]: 0 : visit(overloaded {
2292 : 0 : [](const m::ast::ErrorExpr&) -> void { M_unreachable("no errors at this stage"); },
2293 : 0 : [](const m::ast::Designator&) -> void { /* nothing to be done */ },
2294 : 0 : [](const m::ast::Constant&) -> void { /* nothing to be done */ },
2295 : 0 : [](const m::ast::QueryExpr&) -> void { /* nothing to be done */ },
2296 : 0 : [&min_size_in_bytes](const m::ast::FnApplicationExpr &fn) -> void {
2297 [ # # ]: 0 : if (fn.get_function().is_aggregate())
2298 : 0 : throw visit_skip_subtree(); // skip arguments to already computed aggregate
2299 : 0 : min_size_in_bytes = std::min(min_size_in_bytes, (fn.type()->size() + 7) / 8);
2300 [ # # ]: 0 : if (min_size_in_bytes == 1)
2301 : 0 : throw visit_stop_recursion(); // abort recursion
2302 : 0 : },
2303 : 0 : [&min_size_in_bytes](auto &e) -> void { // i.e. for unary and binary expressions
2304 : 0 : min_size_in_bytes = std::min(min_size_in_bytes, (e.type()->size() + 7) / 8);
2305 [ # # # # ]: 0 : if (min_size_in_bytes == 1)
2306 : 0 : throw visit_stop_recursion(); // abort recursion
2307 : 0 : }
2308 : 0 : }, p.first.get(), m::tag<m::ast::ConstPreOrderExprVisitor>());
2309 : : }
2310 [ # # # # ]: 0 : CodeGenContext::Get().update_num_simd_lanes_preferred(16 / min_size_in_bytes); // set own preference
2311 : :
2312 : : /*----- Execute projection. -----*/
2313 [ # # # # ]: 0 : M.child->get()->execute(std::move(setup), std::move(execute_projection), std::move(teardown));
2314 : 0 : } else {
2315 : : /*----- Execute projection. -----*/
2316 [ # # ]: 0 : setup();
2317 [ # # # # ]: 0 : CodeGenContext::Get().set_num_simd_lanes(1); // since only a single tuple is produced
2318 [ # # ]: 0 : execute_projection();
2319 [ # # ]: 0 : teardown();
2320 : : }
2321 : 0 : }
2322 : :
2323 : :
2324 : : /*======================================================================================================================
2325 : : * Grouping
2326 : : *====================================================================================================================*/
2327 : :
2328 : 0 : ConditionSet HashBasedGrouping::pre_condition(std::size_t child_idx, const std::tuple<const GroupingOperator*>&)
2329 : : {
2330 : 0 : M_insist(child_idx == 0);
2331 : :
2332 : 0 : ConditionSet pre_cond;
2333 : :
2334 : : /*----- Hash-based grouping does not support SIMD. -----*/
2335 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
2336 : :
2337 : 0 : return pre_cond;
2338 [ # # ]: 0 : }
2339 : :
2340 : 0 : double HashBasedGrouping::cost(const Match<HashBasedGrouping> &M)
2341 : : {
2342 : 0 : return 1.5 * M.child->get_matched_root().info().estimated_cardinality;
2343 : : }
2344 : :
2345 : 0 : ConditionSet HashBasedGrouping::post_condition(const Match<HashBasedGrouping>&)
2346 : : {
2347 : 0 : ConditionSet post_cond;
2348 : :
2349 : : /*----- Hash-based grouping does not introduce predication (it is already handled by the hash table). -----*/
2350 [ # # # # ]: 0 : post_cond.add_condition(Predicated(false));
2351 : :
2352 : : /*----- Hash-based grouping does not introduce SIMD. -----*/
2353 [ # # # # ]: 0 : post_cond.add_condition(NoSIMD());
2354 : :
2355 : 0 : return post_cond;
2356 [ # # ]: 0 : }
2357 : :
2358 : 0 : void HashBasedGrouping::execute(const Match<HashBasedGrouping> &M, setup_t setup, pipeline_t pipeline,
2359 : : teardown_t teardown)
2360 : : {
2361 : : // TODO: determine setup
2362 : 0 : const uint64_t AGGREGATES_SIZE_THRESHOLD_IN_BITS =
2363 : 0 : M.use_in_place_values ? std::numeric_limits<uint64_t>::max() : 0;
2364 : :
2365 : 0 : const auto num_keys = M.grouping.group_by().size();
2366 : :
2367 : : /*----- Compute hash table schema and information about aggregates, especially AVG aggregates. -----*/
2368 : 0 : Schema ht_schema;
2369 : : /* Add key(s). */
2370 [ # # ]: 0 : for (std::size_t i = 0; i < num_keys; ++i) {
2371 [ # # # # ]: 0 : auto &e = M.grouping.schema()[i];
2372 [ # # # # ]: 0 : ht_schema.add(e.id, e.type, e.constraints);
2373 : 0 : }
2374 : : /* Add payload. */
2375 [ # # # # : 0 : auto p = compute_aggregate_info(M.grouping.aggregates(), M.grouping.schema(), num_keys);
# # ]
2376 : 0 : const auto &aggregates = p.first;
2377 : 0 : const auto &avg_aggregates = p.second;
2378 : 0 : uint64_t aggregates_size_in_bits = 0;
2379 [ # # ]: 0 : for (auto &info : aggregates) {
2380 [ # # # # ]: 0 : ht_schema.add(info.entry);
2381 [ # # ]: 0 : aggregates_size_in_bits += info.entry.type->size();
2382 : : }
2383 : :
2384 : : /*----- Compute initial capacity of hash table. -----*/
2385 [ # # ]: 0 : uint32_t initial_capacity = compute_initial_ht_capacity(M.grouping, M.load_factor);
2386 : :
2387 : : /*----- Create hash table. -----*/
2388 : 0 : std::unique_ptr<HashTable> ht;
2389 [ # # ]: 0 : std::vector<HashTable::index_t> key_indices(num_keys);
2390 [ # # ]: 0 : std::iota(key_indices.begin(), key_indices.end(), 0);
2391 [ # # ]: 0 : if (M.use_open_addressing_hashing) {
2392 [ # # ]: 0 : if (aggregates_size_in_bits < AGGREGATES_SIZE_THRESHOLD_IN_BITS)
2393 [ # # ]: 0 : ht = std::make_unique<GlobalOpenAddressingInPlaceHashTable>(ht_schema, std::move(key_indices),
2394 : : initial_capacity);
2395 : : else
2396 [ # # ]: 0 : ht = std::make_unique<GlobalOpenAddressingOutOfPlaceHashTable>(ht_schema, std::move(key_indices),
2397 : : initial_capacity);
2398 [ # # ]: 0 : if (M.use_quadratic_probing)
2399 [ # # # # ]: 0 : as<OpenAddressingHashTableBase>(*ht).set_probing_strategy<QuadraticProbing>();
2400 : : else
2401 [ # # # # ]: 0 : as<OpenAddressingHashTableBase>(*ht).set_probing_strategy<LinearProbing>();
2402 : 0 : } else {
2403 [ # # ]: 0 : ht = std::make_unique<GlobalChainedHashTable>(ht_schema, std::move(key_indices), initial_capacity);
2404 : : }
2405 : :
2406 : : /*----- Create child function. -----*/
2407 [ # # # # : 0 : FUNCTION(hash_based_grouping_child_pipeline, void(void)) // create function for pipeline
# # # # ]
2408 : : {
2409 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
2410 : :
2411 : 0 : std::optional<HashTable::entry_t> dummy; ///< *local* dummy slot
2412 : :
2413 [ # # ]: 0 : M.child->execute(
2414 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){
2415 : 0 : ht->setup();
2416 : 0 : ht->set_high_watermark(M.load_factor);
2417 : 0 : dummy.emplace(ht->dummy_entry()); // create dummy slot to ignore NULL values in aggregate computations
2418 : 0 : }),
2419 [ # # ]: 0 : /* pipeline= */ [&](){
2420 : 0 : M_insist(bool(dummy));
2421 : 0 : const auto &env = CodeGenContext::Get().env();
2422 : :
2423 : : /*----- Insert key if not yet done. -----*/
2424 : 0 : std::vector<SQL_t> key;
2425 [ # # ]: 0 : for (auto &p : M.grouping.group_by())
2426 [ # # # # ]: 0 : key.emplace_back(env.compile(p.first.get()));
2427 [ # # ]: 0 : auto [entry, inserted] = ht->try_emplace(std::move(key));
2428 : :
2429 : : /*----- Compute aggregates. -----*/
2430 [ # # ]: 0 : Block init_aggs("hash_based_grouping.init_aggs", false),
2431 [ # # ]: 0 : update_aggs("hash_based_grouping.update_aggs", false),
2432 [ # # ]: 0 : update_avg_aggs("hash_based_grouping.update_avg_aggs", false);
2433 [ # # ]: 0 : for (auto &info : aggregates) {
2434 : 0 : bool is_min = false; ///< flag to indicate whether aggregate function is MIN
2435 [ # # # # : 0 : switch (info.fnid) {
# # ]
2436 : : default:
2437 [ # # ]: 0 : M_unreachable("unsupported aggregate function");
2438 : : case m::Function::FN_MIN:
2439 : 0 : is_min = true; // set flag and delegate to MAX case
2440 : : case m::Function::FN_MAX: {
2441 [ # # ]: 0 : M_insist(info.args.size() == 1,
2442 : : "MIN and MAX aggregate functions expect exactly one argument");
2443 : 0 : const auto &arg = *info.args[0];
2444 [ # # ]: 0 : std::visit(overloaded {
2445 : 0 : [&]<sql_type _T>(HashTable::reference_t<_T> &&r) -> void
2446 : : requires (not (std::same_as<_T, _Boolx1> or std::same_as<_T, NChar>)) {
2447 : : using type = typename _T::type;
2448 : : using T = PrimitiveExpr<type>;
2449 : :
2450 : 0 : auto _arg = env.compile(arg);
2451 [ # # # # : 0 : _T _new_val = convert<_T>(_arg);
# # # # #
# # # ]
2452 : :
2453 [ # # # # : 0 : BLOCK_OPEN(init_aggs) {
# # # # #
# # # ]
2454 [ # # # # : 0 : auto [val_, is_null] = _new_val.clone().split();
# # # # #
# # # # #
# # # # #
# # # #
# ]
2455 [ # # # # : 0 : T val(val_); // due to structured binding and lambda closure
# # # # #
# # # ]
2456 [ # # # # : 0 : IF (is_null) {
# # # # #
# # # # #
# # # # #
# # # #
# ]
2457 [ # # # # : 0 : auto neutral = is_min ? T(std::numeric_limits<type>::max())
# # # # #
# # # ]
2458 : 0 : : T(std::numeric_limits<type>::lowest());
2459 [ # # # # : 0 : r.clone().set_value(neutral); // initialize with neutral element +inf or -inf
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2460 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # #
# ]
2461 [ # # # # : 0 : r.clone().set_null(); // first value is NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
2462 [ # # # # : 0 : } ELSE {
# # # # #
# # # ]
2463 [ # # # # : 0 : r.clone().set_value(val); // initialize with first value
# # # # #
# # # # #
# # # # #
# # # #
# ]
2464 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
2465 [ # # # # : 0 : r.clone().set_not_null(); // first value is not NULL
# # # # #
# # # ]
2466 : 0 : };
2467 : 0 : }
2468 [ # # # # : 0 : BLOCK_OPEN(update_aggs) {
# # # # #
# # # ]
2469 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # ]
2470 : : M_insist_no_ternary_logic();
2471 [ # # # # : 0 : auto [new_val_, new_val_is_null_] = _new_val.split();
# # # # #
# # # ]
2472 [ # # # # : 0 : auto [old_min_max_, old_min_max_is_null] = _T(r.clone()).split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2473 [ # # # # : 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # ]
2474 : :
2475 [ # # # # : 0 : auto chosen_r = Select(new_val_is_null, dummy->extract<_T>(info.entry.id),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2476 [ # # # # : 0 : r.clone());
# # # # #
# # # ]
2477 : : if constexpr (std::floating_point<type>) {
2478 [ # # # # : 0 : chosen_r.set_value(
# # # # #
# # # # #
# # # # #
# ]
2479 [ # # # # : 0 : is_min ? min(old_min_max_, new_val_) // update old min with new value
# # # # ]
2480 [ # # # # ]: 0 : : max(old_min_max_, new_val_) // update old max with new value
2481 : : ); // if new value is NULL, only dummy is written
2482 : : } else {
2483 [ # # # # : 0 : const Var<T> new_val(new_val_),
# # # # ]
2484 [ # # # # : 0 : old_min_max(old_min_max_); // due to multiple uses
# # # # ]
2485 [ # # # # : 0 : auto cmp = is_min ? new_val < old_min_max : new_val > old_min_max;
# # # # #
# # # # #
# # # # #
# # # #
# ]
2486 : : #if 1
2487 [ # # # # : 0 : chosen_r.set_value(
# # # # ]
2488 [ # # # # : 0 : Select(cmp,
# # # # ]
2489 : : new_val, // update to new value
2490 : : old_min_max) // do not update
2491 : : ); // if new value is NULL, only dummy is written
2492 : : #else
2493 : : IF (cmp) {
2494 : : r.set_value(new_val);
2495 : : };
2496 : : #endif
2497 : 0 : }
2498 [ # # # # : 0 : r.set_null_bit(
# # # # #
# # # ]
2499 [ # # # # : 0 : old_min_max_is_null and new_val_is_null // MIN/MAX is NULL iff all values are NULL
# # # # #
# # # ]
2500 : : );
2501 : 0 : } else {
2502 [ # # # # : 0 : auto new_val_ = _new_val.insist_not_null();
# # # # #
# # # ]
2503 [ # # # # : 0 : auto old_min_max_ = _T(r.clone()).insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2504 : : if constexpr (std::floating_point<type>) {
2505 [ # # # # : 0 : r.set_value(
# # # # #
# # # # #
# # # # #
# ]
2506 [ # # # # : 0 : is_min ? min(old_min_max_, new_val_) // update old min with new value
# # # # ]
2507 [ # # # # ]: 0 : : max(old_min_max_, new_val_) // update old max with new value
2508 : : );
2509 : : } else {
2510 [ # # # # : 0 : const Var<T> new_val(new_val_),
# # # # ]
2511 [ # # # # : 0 : old_min_max(old_min_max_); // due to multiple uses
# # # # ]
2512 [ # # # # : 0 : auto cmp = is_min ? new_val < old_min_max : new_val > old_min_max;
# # # # #
# # # # #
# # # # #
# # # #
# ]
2513 : : #if 1
2514 [ # # # # : 0 : r.set_value(
# # # # ]
2515 [ # # # # : 0 : Select(cmp,
# # # # ]
2516 : : new_val, // update to new value
2517 : : old_min_max) // do not update
2518 : : );
2519 : : #else
2520 : : IF (cmp) {
2521 : : r.set_value(new_val);
2522 : : };
2523 : : #endif
2524 : 0 : }
2525 : : /* do not update NULL bit since it is already set to `false` */
2526 : 0 : }
2527 : : }
2528 : 0 : },
2529 : 0 : []<sql_type _T>(HashTable::reference_t<_T>&&) -> void
2530 : : requires std::same_as<_T,_Boolx1> or std::same_as<_T, NChar> {
2531 : 0 : M_unreachable("invalid type");
2532 : : },
2533 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
2534 [ # # ]: 0 : }, entry.extract(info.entry.id));
2535 : 0 : break;
2536 : : }
2537 : : case m::Function::FN_AVG: {
2538 [ # # ]: 0 : auto it = avg_aggregates.find(info.entry.id);
2539 [ # # ]: 0 : M_insist(it != avg_aggregates.end());
2540 : 0 : const auto &avg_info = it->second;
2541 [ # # ]: 0 : M_insist(avg_info.compute_running_avg,
2542 : : "AVG aggregate may only occur for running average computations");
2543 [ # # ]: 0 : M_insist(info.args.size() == 1, "AVG aggregate function expects exactly one argument");
2544 : 0 : const auto &arg = *info.args[0];
2545 : :
2546 [ # # ]: 0 : auto r = entry.extract<_Doublex1>(info.entry.id);
2547 [ # # ]: 0 : auto _arg = env.compile(arg);
2548 [ # # ]: 0 : _Doublex1 _new_val = convert<_Doublex1>(_arg);
2549 : :
2550 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
2551 [ # # # # ]: 0 : auto [val_, is_null] = _new_val.clone().split();
2552 [ # # ]: 0 : Doublex1 val(val_); // due to structured binding and lambda closure
2553 [ # # ]: 0 : IF (is_null) {
2554 [ # # # # ]: 0 : r.clone().set_value(Doublex1(0.0)); // initialize with neutral element 0
2555 [ # # ]: 0 : if (info.entry.nullable())
2556 [ # # ]: 0 : r.clone().set_null(); // first value is NULL
2557 [ # # ]: 0 : } ELSE {
2558 [ # # # # ]: 0 : r.clone().set_value(val); // initialize with first value
2559 [ # # ]: 0 : if (info.entry.nullable())
2560 [ # # ]: 0 : r.clone().set_not_null(); // first value is not NULL
2561 : 0 : };
2562 : 0 : }
2563 [ # # ]: 0 : BLOCK_OPEN(update_avg_aggs) {
2564 : : /* Compute AVG as iterative mean as described in Knuth, The Art of Computer Programming
2565 : : * Vol 2, section 4.2.2. */
2566 [ # # ]: 0 : if (_new_val.can_be_null()) {
2567 : : M_insist_no_ternary_logic();
2568 [ # # ]: 0 : auto [new_val, new_val_is_null_] = _new_val.split();
2569 [ # # # # : 0 : auto [old_avg_, old_avg_is_null] = _Doublex1(r.clone()).split();
# # ]
2570 [ # # ]: 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
2571 [ # # ]: 0 : const Var<Doublex1> old_avg(old_avg_); // due to multiple uses
2572 : :
2573 [ # # ]: 0 : auto delta_absolute = new_val - old_avg;
2574 [ # # # # : 0 : auto running_count = _I64x1(entry.get<_I64x1>(avg_info.running_count)).insist_not_null();
# # ]
2575 [ # # # # ]: 0 : auto delta_relative = delta_absolute / running_count.to<double>();
2576 : :
2577 [ # # # # : 0 : auto chosen_r = Select(new_val_is_null, dummy->extract<_Doublex1>(info.entry.id),
# # ]
2578 [ # # ]: 0 : r.clone());
2579 [ # # ]: 0 : chosen_r.set_value(
2580 [ # # ]: 0 : old_avg + delta_relative // update old average with new value
2581 : : ); // if new value is NULL, only dummy is written
2582 [ # # ]: 0 : r.set_null_bit(
2583 [ # # ]: 0 : old_avg_is_null and new_val_is_null // AVG is NULL iff all values are NULL
2584 : : );
2585 : 0 : } else {
2586 [ # # ]: 0 : auto new_val = _new_val.insist_not_null();
2587 [ # # # # : 0 : auto old_avg_ = _Doublex1(r.clone()).insist_not_null();
# # ]
2588 [ # # ]: 0 : const Var<Doublex1> old_avg(old_avg_); // due to multiple uses
2589 : :
2590 [ # # ]: 0 : auto delta_absolute = new_val - old_avg;
2591 [ # # # # : 0 : auto running_count = _I64x1(entry.get<_I64x1>(avg_info.running_count)).insist_not_null();
# # ]
2592 [ # # # # ]: 0 : auto delta_relative = delta_absolute / running_count.to<double>();
2593 [ # # ]: 0 : r.set_value(
2594 [ # # ]: 0 : old_avg + delta_relative // update old average with new value
2595 : : );
2596 : : /* do not update NULL bit since it is already set to `false` */
2597 : 0 : }
2598 : : }
2599 : : break;
2600 : 0 : }
2601 : : case m::Function::FN_SUM: {
2602 [ # # ]: 0 : M_insist(info.args.size() == 1, "SUM aggregate function expects exactly one argument");
2603 : 0 : const auto &arg = *info.args[0];
2604 [ # # ]: 0 : std::visit(overloaded {
2605 : 0 : [&]<sql_type _T>(HashTable::reference_t<_T> &&r) -> void
2606 : : requires (not (std::same_as<_T, _Boolx1> or std::same_as<_T, NChar>)) {
2607 : : using type = typename _T::type;
2608 : : using T = PrimitiveExpr<type>;
2609 : :
2610 : 0 : auto _arg = env.compile(arg);
2611 [ # # # # : 0 : _T _new_val = convert<_T>(_arg);
# # # # #
# # # ]
2612 : :
2613 [ # # # # : 0 : BLOCK_OPEN(init_aggs) {
# # # # #
# # # ]
2614 [ # # # # : 0 : auto [val_, is_null] = _new_val.clone().split();
# # # # #
# # # # #
# # # # #
# # # #
# ]
2615 [ # # # # : 0 : T val(val_); // due to structured binding and lambda closure
# # # # #
# # # ]
2616 [ # # # # : 0 : IF (is_null) {
# # # # #
# # # ]
2617 [ # # # # : 0 : r.clone().set_value(T(type(0))); // initialize with neutral element 0
# # # # #
# # # # #
# # # # #
# # # #
# ]
2618 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
2619 [ # # # # : 0 : r.clone().set_null(); // first value is NULL
# # # # #
# # # ]
2620 [ # # # # : 0 : } ELSE {
# # # # #
# # # ]
2621 [ # # # # : 0 : r.clone().set_value(val); // initialize with first value
# # # # #
# # # # #
# # # # #
# # # #
# ]
2622 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
2623 [ # # # # : 0 : r.clone().set_not_null(); // first value is not NULL
# # # # #
# # # ]
2624 : 0 : };
2625 : 0 : }
2626 [ # # # # : 0 : BLOCK_OPEN(update_aggs) {
# # # # #
# # # ]
2627 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # ]
2628 : : M_insist_no_ternary_logic();
2629 [ # # # # : 0 : auto [new_val, new_val_is_null_] = _new_val.split();
# # # # #
# # # ]
2630 [ # # # # : 0 : auto [old_sum, old_sum_is_null] = _T(r.clone()).split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2631 [ # # # # : 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # ]
2632 : :
2633 [ # # # # : 0 : auto chosen_r = Select(new_val_is_null, dummy->extract<_T>(info.entry.id),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2634 [ # # # # : 0 : r.clone());
# # # # #
# # # ]
2635 [ # # # # : 0 : chosen_r.set_value(
# # # # #
# # # ]
2636 [ # # # # : 0 : old_sum + new_val // add new value to old sum
# # # # #
# # # # #
# # # # #
# # # #
# ]
2637 : : ); // if new value is NULL, only dummy is written
2638 [ # # # # : 0 : r.set_null_bit(
# # # # #
# # # ]
2639 [ # # # # : 0 : old_sum_is_null and new_val_is_null // SUM is NULL iff all values are NULL
# # # # #
# # # ]
2640 : : );
2641 : 0 : } else {
2642 [ # # # # : 0 : auto new_val = _new_val.insist_not_null();
# # # # #
# # # ]
2643 [ # # # # : 0 : auto old_sum = _T(r.clone()).insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2644 [ # # # # : 0 : r.set_value(
# # # # #
# # # ]
2645 [ # # # # : 0 : old_sum + new_val // add new value to old sum
# # # # #
# # # # #
# # # # #
# # # #
# ]
2646 : : );
2647 : : /* do not update NULL bit since it is already set to `false` */
2648 : 0 : }
2649 : : }
2650 : 0 : },
2651 : 0 : []<sql_type _T>(HashTable::reference_t<_T>&&) -> void
2652 : : requires std::same_as<_T,_Boolx1> or std::same_as<_T, NChar> {
2653 : 0 : M_unreachable("invalid type");
2654 : : },
2655 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
2656 [ # # ]: 0 : }, entry.extract(info.entry.id));
2657 : 0 : break;
2658 : : }
2659 : : case m::Function::FN_COUNT: {
2660 [ # # ]: 0 : M_insist(info.args.size() <= 1, "COUNT aggregate function expects at most one argument");
2661 : :
2662 [ # # ]: 0 : auto r = entry.get<_I64x1>(info.entry.id); // do not extract to be able to access for AVG case
2663 : :
2664 [ # # ]: 0 : if (info.args.empty()) {
2665 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
2666 [ # # # # : 0 : r.clone() = _I64x1(1); // initialize with 1 (for first value)
# # ]
2667 : : }
2668 [ # # ]: 0 : BLOCK_OPEN(update_aggs) {
2669 [ # # # # : 0 : auto old_count = _I64x1(r.clone()).insist_not_null();
# # ]
2670 [ # # ]: 0 : r.set_value(
2671 [ # # ]: 0 : old_count + int64_t(1) // increment old count by 1
2672 : : );
2673 : : /* do not update NULL bit since it is already set to `false` */
2674 : 0 : }
2675 : 0 : } else {
2676 : 0 : const auto &arg = *info.args[0];
2677 : :
2678 [ # # ]: 0 : auto _arg = env.compile(arg);
2679 [ # # # # ]: 0 : I64x1 new_val_not_null = not_null(_arg).to<int64_t>();
2680 : :
2681 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
2682 [ # # # # : 0 : r.clone() = _I64x1(new_val_not_null.clone()); // initialize with 1 iff first value is present
# # # # ]
2683 : : }
2684 [ # # ]: 0 : BLOCK_OPEN(update_aggs) {
2685 [ # # # # : 0 : auto old_count = _I64x1(r.clone()).insist_not_null();
# # ]
2686 [ # # ]: 0 : r.set_value(
2687 [ # # # # ]: 0 : old_count + new_val_not_null // increment old count by 1 iff new value is present
2688 : : );
2689 : : /* do not update NULL bit since it is already set to `false` */
2690 : 0 : }
2691 : 0 : }
2692 : : break;
2693 : 0 : }
2694 : : }
2695 : : }
2696 : :
2697 : : /*----- If group has been inserted, initialize aggregates. Otherwise, update them. -----*/
2698 [ # # ]: 0 : IF (inserted) {
2699 : 0 : init_aggs.attach_to_current();
2700 : 0 : } ELSE {
2701 : 0 : update_aggs.attach_to_current();
2702 : 0 : update_avg_aggs.attach_to_current(); // after others to ensure that running count is incremented before
2703 : 0 : };
2704 : 0 : },
2705 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ ht->teardown(); })
2706 : : );
2707 : 0 : }
2708 [ # # ]: 0 : hash_based_grouping_child_pipeline(); // call child function
2709 : :
2710 [ # # # # ]: 0 : auto &env = CodeGenContext::Get().env();
2711 : :
2712 : : /*----- Process each computed group. -----*/
2713 [ # # # # ]: 0 : setup_t(std::move(setup), [&](){ ht->setup(); })();
2714 [ # # # # ]: 0 : ht->for_each([&, pipeline=std::move(pipeline)](HashTable::const_entry_t entry){
2715 : : /*----- Compute key schema to detect duplicated keys. -----*/
2716 : 0 : Schema key_schema;
2717 [ # # ]: 0 : for (std::size_t i = 0; i < num_keys; ++i) {
2718 [ # # ]: 0 : auto &e = M.grouping.schema()[i];
2719 [ # # # # ]: 0 : key_schema.add(e.id, e.type, e.constraints);
2720 : 0 : }
2721 : :
2722 : : /*----- Add computed group tuples to current environment. ----*/
2723 [ # # # # ]: 0 : for (auto &e : M.grouping.schema().deduplicate()) {
2724 : : try {
2725 [ # # ]: 0 : key_schema.find(e.id);
2726 [ # # ]: 0 : } catch (invalid_argument&) {
2727 : : continue; // skip duplicated keys since they must not be used afterwards
2728 [ # # ]: 0 : }
2729 : :
2730 [ # # # # ]: 0 : if (auto it = avg_aggregates.find(e.id);
2731 [ # # ]: 0 : it != avg_aggregates.end() and not it->second.compute_running_avg)
2732 : : { // AVG aggregates which is not yet computed, divide computed sum with computed count
2733 : 0 : auto &avg_info = it->second;
2734 [ # # ]: 0 : auto sum = std::visit(overloaded {
2735 : 0 : [&]<sql_type T>(HashTable::const_reference_t<T> &&r) -> _Doublex1
2736 : : requires (std::same_as<T, _I64x1> or std::same_as<T, _Doublex1>) {
2737 [ # # # # ]: 0 : return T(r).template to<double>();
2738 : 0 : },
2739 : 0 : [](auto&&) -> _Doublex1 { M_unreachable("invalid type"); },
2740 : 0 : [](std::monostate&&) -> _Doublex1 { M_unreachable("invalid reference"); },
2741 [ # # ]: 0 : }, entry.get(avg_info.sum));
2742 [ # # # # : 0 : auto count = _I64x1(entry.get<_I64x1>(avg_info.running_count)).insist_not_null().to<double>();
# # # # ]
2743 [ # # ]: 0 : auto avg = sum / count;
2744 [ # # ]: 0 : if (avg.can_be_null()) {
2745 [ # # ]: 0 : _Var<Doublex1> var(avg); // introduce variable s.t. uses only load from it
2746 [ # # # # : 0 : env.add(e.id, var);
# # ]
2747 : 0 : } else {
2748 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
2749 [ # # # # ]: 0 : Var<Doublex1> var(avg.insist_not_null());
2750 [ # # # # : 0 : env.add(e.id, _Doublex1(var));
# # # # ]
2751 : 0 : }
2752 : 0 : } else { // part of key or already computed aggregate
2753 [ # # ]: 0 : std::visit(overloaded {
2754 : 0 : [&]<typename T>(HashTable::const_reference_t<Expr<T>> &&r) -> void {
2755 : 0 : Expr<T> value = r;
2756 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # #
# ]
2757 [ # # # # : 0 : Var<Expr<T>> var(value); // introduce variable s.t. uses only load from it
# # # # #
# # # #
# ]
2758 [ # # # # : 0 : env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2759 : 0 : } else {
2760 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
2761 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(value.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2762 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2763 : 0 : }
2764 : 0 : },
2765 : 0 : [&](HashTable::const_reference_t<NChar> &&r) -> void {
2766 : 0 : NChar value(r);
2767 [ # # # # ]: 0 : Var<Ptr<Charx1>> var(value.val()); // introduce variable s.t. uses only load from it
2768 [ # # # # : 0 : env.add(e.id, NChar(var, value.can_be_null(), value.length(),
# # # # #
# ]
2769 : 0 : value.guarantees_terminating_nul()));
2770 : 0 : },
2771 : 0 : [](std::monostate&&) -> void { M_unreachable("invalid reference"); },
2772 [ # # ]: 0 : }, entry.get(e.id)); // do not extract to be able to access for not-yet-computed AVG aggregates
2773 : : }
2774 : : }
2775 : :
2776 : : /*----- Resume pipeline. -----*/
2777 [ # # ]: 0 : pipeline();
2778 : 0 : });
2779 [ # # # # ]: 0 : teardown_t(std::move(teardown), [&](){ ht->teardown(); })();
2780 : 0 : }
2781 : :
2782 : 0 : ConditionSet OrderedGrouping::pre_condition(
2783 : : std::size_t child_idx,
2784 : : const std::tuple<const GroupingOperator*> &partial_inner_nodes)
2785 : : {
2786 : 0 : M_insist(child_idx == 0);
2787 : :
2788 : 0 : ConditionSet pre_cond;
2789 : :
2790 : : /*----- Ordered grouping needs the data sorted on the grouping key (in either order). -----*/
2791 : 0 : Sortedness::order_t orders;
2792 [ # # # # ]: 0 : for (auto &p : std::get<0>(partial_inner_nodes)->group_by()) {
2793 [ # # ]: 0 : Schema::Identifier id(p.first);
2794 [ # # # # : 0 : if (orders.find(id) == orders.cend())
# # ]
2795 [ # # # # ]: 0 : orders.add(std::move(id), Sortedness::O_UNDEF);
2796 : 0 : }
2797 [ # # # # ]: 0 : pre_cond.add_condition(Sortedness(std::move(orders)));
2798 : :
2799 : : /*----- Ordered grouping does not support SIMD. -----*/
2800 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
2801 : :
2802 : 0 : return pre_cond;
2803 [ # # ]: 0 : }
2804 : :
2805 : 0 : double OrderedGrouping::cost(const Match<OrderedGrouping> &M)
2806 : : {
2807 : 0 : return 1.0 * M.child->get_matched_root().info().estimated_cardinality;
2808 : : }
2809 : :
2810 : 0 : ConditionSet OrderedGrouping::adapt_post_condition(const Match<OrderedGrouping> &M, const ConditionSet &post_cond_child)
2811 : : {
2812 : 0 : ConditionSet post_cond;
2813 : :
2814 : : /*----- Ordered grouping does not introduce predication. -----*/
2815 [ # # # # ]: 0 : post_cond.add_condition(Predicated(false));
2816 : :
2817 : : /*----- Preserve order of child for grouping keys. -----*/
2818 : 0 : Sortedness::order_t orders;
2819 [ # # ]: 0 : const auto &sortedness_child = post_cond_child.get_condition<Sortedness>();
2820 [ # # # # ]: 0 : for (auto &[expr, alias] : M.grouping.group_by()) {
2821 [ # # # # : 0 : auto it = sortedness_child.orders().find(Schema::Identifier(expr));
# # ]
2822 [ # # # # : 0 : M_insist(it != sortedness_child.orders().cend());
# # ]
2823 [ # # # # : 0 : Schema::Identifier id = alias.has_value() ? Schema::Identifier(alias.assert_not_none())
# # # # #
# # # ]
2824 [ # # ]: 0 : : Schema::Identifier(expr);
2825 [ # # # # : 0 : if (orders.find(id) == orders.cend())
# # ]
2826 [ # # # # ]: 0 : orders.add(std::move(id), it->second); // drop duplicate since it must not be used afterwards
2827 : 0 : }
2828 [ # # # # ]: 0 : post_cond.add_condition(Sortedness(std::move(orders)));
2829 : :
2830 : : /*----- Ordered grouping does not introduce SIMD. -----*/
2831 [ # # # # ]: 0 : post_cond.add_condition(NoSIMD());
2832 : :
2833 : 0 : return post_cond;
2834 [ # # ]: 0 : }
2835 : :
2836 : 0 : void OrderedGrouping::execute(const Match<OrderedGrouping> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
2837 : : {
2838 : 0 : Environment results; ///< stores current result tuple
2839 [ # # ]: 0 : const auto num_keys = M.grouping.group_by().size();
2840 : :
2841 : : /*----- Compute key schema to detect duplicated keys. -----*/
2842 : 0 : Schema key_schema;
2843 [ # # ]: 0 : for (std::size_t i = 0; i < num_keys; ++i) {
2844 [ # # # # ]: 0 : auto &e = M.grouping.schema()[i];
2845 [ # # # # ]: 0 : key_schema.add(e.id, e.type, e.constraints);
2846 : 0 : }
2847 : :
2848 : : /*----- Compute information about aggregates, especially about AVG aggregates. -----*/
2849 [ # # # # : 0 : auto p = compute_aggregate_info(M.grouping.aggregates(), M.grouping.schema(), num_keys);
# # ]
2850 : 0 : const auto &aggregates = p.first;
2851 : 0 : const auto &avg_aggregates = p.second;
2852 : :
2853 : : /*----- Forward declare function to emit a group tuple in the current environment and resume the pipeline. -----*/
2854 [ # # ]: 0 : FunctionProxy<void(void)> emit_group_and_resume_pipeline("emit_group_and_resume_pipeline");
2855 : :
2856 : 0 : std::optional<Var<Boolx1>> first_iteration; ///< variable to *locally* check for first iteration
2857 : : ///> *global* flag backup since the following code may be called multiple times
2858 [ # # ]: 0 : Global<Boolx1> first_iteration_backup(true);
2859 : :
2860 : : using agg_t = agg_t_<false>;
2861 : : using agg_backup_t = agg_t_<true>;
2862 [ # # # # : 0 : agg_t agg_values[aggregates.size()]; ///< *local* values of the computed aggregates
# # ]
2863 [ # # # # : 0 : agg_backup_t agg_value_backups[aggregates.size()]; ///< *global* value backups of the computed aggregates
# # ]
2864 : :
2865 : : using key_t = key_t_<false>;
2866 : : using key_backup_t = key_t_<true>;
2867 [ # # # # : 0 : key_t key_values[num_keys]; ///< *local* values of the computed keys
# # ]
2868 [ # # # # : 0 : key_backup_t key_value_backups[num_keys]; ///< *global* value backups of the computed keys
# # ]
2869 : :
2870 : 0 : auto store_locals_to_globals = [&](){
2871 : : /*----- Store local aggregate values to globals to access them in other function. -----*/
2872 [ # # ]: 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
2873 : 0 : auto &info = aggregates[idx];
2874 : :
2875 [ # # # # : 0 : switch (info.fnid) {
# ]
2876 : : default:
2877 : 0 : M_unreachable("unsupported aggregate function");
2878 : : case m::Function::FN_MIN:
2879 : : case m::Function::FN_MAX: {
2880 : 0 : auto min_max = [&]<typename T>() {
2881 : 0 : auto &[min_max, is_null] = *M_notnull((
2882 : : std::get_if<std::pair<Var<PrimitiveExpr<T>>, std::optional<Var<Boolx1>>>>(&agg_values[idx])
2883 : : ));
2884 : 0 : auto &[min_max_backup, is_null_backup] = *M_notnull((
2885 : : std::get_if<std::pair<Global<PrimitiveExpr<T>>,
2886 : : std::optional<Global<Boolx1>>>>(&agg_value_backups[idx])
2887 : : ));
2888 : 0 : M_insist(bool(is_null) == bool(is_null_backup));
2889 : :
2890 : 0 : min_max_backup = min_max;
2891 [ # # # # : 0 : if (is_null)
# # # # #
# # # ]
2892 : 0 : *is_null_backup = *is_null;
2893 : 0 : };
2894 : 0 : auto &n = as<const Numeric>(*info.entry.type);
2895 [ # # # ]: 0 : switch (n.kind) {
2896 : : case Numeric::N_Int:
2897 : : case Numeric::N_Decimal:
2898 [ # # # # : 0 : switch (n.size()) {
# ]
2899 : 0 : default: M_unreachable("invalid size");
2900 : 0 : case 8: min_max.template operator()<int8_t >(); break;
2901 : 0 : case 16: min_max.template operator()<int16_t>(); break;
2902 : 0 : case 32: min_max.template operator()<int32_t>(); break;
2903 : 0 : case 64: min_max.template operator()<int64_t>(); break;
2904 : : }
2905 : 0 : break;
2906 : : case Numeric::N_Float:
2907 [ # # ]: 0 : if (n.size() <= 32)
2908 : 0 : min_max.template operator()<float>();
2909 : : else
2910 : 0 : min_max.template operator()<double>();
2911 : 0 : }
2912 : 0 : break;
2913 : : }
2914 : : case m::Function::FN_AVG: {
2915 : 0 : auto &[avg, is_null] = *M_notnull((
2916 : : std::get_if<std::pair<Var<Doublex1>, std::optional<Var<Boolx1>>>>(&agg_values[idx])
2917 : : ));
2918 : 0 : auto &[avg_backup, is_null_backup] = *M_notnull((
2919 : : std::get_if<std::pair<Global<Doublex1>, std::optional<Global<Boolx1>>>>(&agg_value_backups[idx])
2920 : : ));
2921 : 0 : M_insist(bool(is_null) == bool(is_null_backup));
2922 : :
2923 : 0 : avg_backup = avg;
2924 [ # # ]: 0 : if (is_null)
2925 : 0 : *is_null_backup = *is_null;
2926 : :
2927 : 0 : break;
2928 : : }
2929 : : case m::Function::FN_SUM: {
2930 : 0 : M_insist(info.args.size() == 1, "SUM aggregate function expects exactly one argument");
2931 : 0 : const auto &arg = *info.args[0];
2932 : :
2933 : 0 : auto sum = [&]<typename T>() {
2934 : 0 : auto &[sum, is_null] = *M_notnull((
2935 : : std::get_if<std::pair<Var<PrimitiveExpr<T>>, std::optional<Var<Boolx1>>>>(&agg_values[idx])
2936 : : ));
2937 : 0 : auto &[sum_backup, is_null_backup] = *M_notnull((
2938 : : std::get_if<std::pair<Global<PrimitiveExpr<T>>,
2939 : : std::optional<Global<Boolx1>>>>(&agg_value_backups[idx])
2940 : : ));
2941 : 0 : M_insist(bool(is_null) == bool(is_null_backup));
2942 : :
2943 : 0 : sum_backup = sum;
2944 [ # # # # : 0 : if (is_null)
# # # # #
# # # ]
2945 : 0 : *is_null_backup = *is_null;
2946 : 0 : };
2947 : 0 : auto &n = as<const Numeric>(*info.entry.type);
2948 [ # # # ]: 0 : switch (n.kind) {
2949 : : case Numeric::N_Int:
2950 : : case Numeric::N_Decimal:
2951 [ # # # # : 0 : switch (n.size()) {
# ]
2952 : 0 : default: M_unreachable("invalid size");
2953 : 0 : case 8: sum.template operator()<int8_t >(); break;
2954 : 0 : case 16: sum.template operator()<int16_t>(); break;
2955 : 0 : case 32: sum.template operator()<int32_t>(); break;
2956 : 0 : case 64: sum.template operator()<int64_t>(); break;
2957 : : }
2958 : 0 : break;
2959 : : case Numeric::N_Float:
2960 [ # # ]: 0 : if (n.size() <= 32)
2961 : 0 : sum.template operator()<float>();
2962 : : else
2963 : 0 : sum.template operator()<double>();
2964 : 0 : }
2965 : 0 : break;
2966 : : }
2967 : : case m::Function::FN_COUNT: {
2968 : 0 : auto &count = *M_notnull(std::get_if<Var<I64x1>>(&agg_values[idx]));
2969 : 0 : auto &count_backup = *M_notnull(std::get_if<Global<I64x1>>(&agg_value_backups[idx]));
2970 : :
2971 : 0 : count_backup = count;
2972 : :
2973 : 0 : break;
2974 : : }
2975 : : }
2976 : 0 : }
2977 : :
2978 : : /*----- Store local key values to globals to access them in other function. -----*/
2979 : 0 : auto store = [&]<typename T>(std::size_t idx) {
2980 : 0 : auto &[key, is_null] = *M_notnull((
2981 : : std::get_if<std::pair<Var<PrimitiveExpr<T>>, std::optional<Var<Boolx1>>>>(&key_values[idx])
2982 : : ));
2983 : 0 : auto &[key_backup, is_null_backup] = *M_notnull((
2984 : : std::get_if<std::pair<Global<PrimitiveExpr<T>>, std::optional<Global<Boolx1>>>>(&key_value_backups[idx])
2985 : : ));
2986 : 0 : M_insist(bool(is_null) == bool(is_null_backup));
2987 : :
2988 : 0 : key_backup = key;
2989 [ # # # # : 0 : if (is_null)
# # # # #
# # # #
# ]
2990 : 0 : *is_null_backup = *is_null;
2991 : 0 : };
2992 [ # # ]: 0 : for (std::size_t idx = 0; idx < num_keys; ++idx) {
2993 : 0 : visit(overloaded{
2994 : 0 : [&](const Boolean&) { store.template operator()<bool>(idx); },
2995 : 0 : [&](const Numeric &n) {
2996 [ # # # ]: 0 : switch (n.kind) {
2997 : : case Numeric::N_Int:
2998 : : case Numeric::N_Decimal:
2999 [ # # # # : 0 : switch (n.size()) {
# ]
3000 : 0 : default: M_unreachable("invalid size");
3001 : 0 : case 8: store.template operator()<int8_t >(idx); break;
3002 : 0 : case 16: store.template operator()<int16_t>(idx); break;
3003 : 0 : case 32: store.template operator()<int32_t>(idx); break;
3004 : 0 : case 64: store.template operator()<int64_t>(idx); break;
3005 : : }
3006 : 0 : break;
3007 : : case Numeric::N_Float:
3008 [ # # ]: 0 : if (n.size() <= 32)
3009 : 0 : store.template operator()<float>(idx);
3010 : : else
3011 : 0 : store.template operator()<double>(idx);
3012 : 0 : }
3013 : 0 : },
3014 : 0 : [&](const CharacterSequence &cs) {
3015 : 0 : auto &key = *M_notnull(std::get_if<Var<Ptr<Charx1>>>(&key_values[idx]));
3016 : 0 : auto &key_backup = *M_notnull(std::get_if<Global<Ptr<Charx1>>>(&key_value_backups[idx]));
3017 : :
3018 : 0 : key_backup = key;
3019 : 0 : },
3020 : 0 : [&](const Date&) { store.template operator()<int32_t>(idx); },
3021 : 0 : [&](const DateTime&) { store.template operator()<int64_t>(idx); },
3022 : 0 : [](auto&&) { M_unreachable("invalid type"); },
3023 : 0 : }, *M.grouping.schema()[idx].type);
3024 : 0 : }
3025 : 0 : };
3026 : :
3027 [ # # ]: 0 : M.child->execute(
3028 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){
3029 : 0 : first_iteration.emplace(first_iteration_backup);
3030 : :
3031 : : /*----- Initialize aggregates and their backups. -----*/
3032 [ # # ]: 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
3033 : 0 : auto &info = aggregates[idx];
3034 : 0 : const bool nullable = info.entry.nullable();
3035 : :
3036 : 0 : bool is_min = false; ///< flag to indicate whether aggregate function is MIN
3037 [ # # # # : 0 : switch (info.fnid) {
# # ]
3038 : : default:
3039 : 0 : M_unreachable("unsupported aggregate function");
3040 : : case m::Function::FN_MIN:
3041 : 0 : is_min = true; // set flag and delegate to MAX case
3042 : : case m::Function::FN_MAX: {
3043 : 0 : auto min_max = [&]<typename T>() {
3044 : 0 : auto neutral = is_min ? std::numeric_limits<T>::max()
3045 : 0 : : std::numeric_limits<T>::lowest();
3046 : :
3047 : 0 : Var<PrimitiveExpr<T>> min_max;
3048 [ # # # # : 0 : Global<PrimitiveExpr<T>> min_max_backup(neutral); // initialize with neutral element +inf or -inf
# # # # #
# # # ]
3049 : 0 : std::optional<Var<Boolx1>> is_null;
3050 : 0 : std::optional<Global<Boolx1>> is_null_backup;
3051 : :
3052 : : /*----- Set local aggregate variables to global backups. -----*/
3053 [ # # # # : 0 : min_max = min_max_backup;
# # # # #
# # # ]
3054 [ # # # # : 0 : if (nullable) {
# # # # #
# # # ]
3055 [ # # # # : 0 : is_null_backup.emplace(true); // MIN/MAX is initially NULL
# # # # #
# # # ]
3056 [ # # # # : 0 : is_null.emplace(*is_null_backup);
# # # # #
# # # ]
3057 : 0 : }
3058 : :
3059 : : /*----- Add global aggregate to result environment to access it in other function. -----*/
3060 [ # # # # : 0 : if (nullable)
# # # # #
# # # ]
3061 [ # # # # : 0 : results.add(info.entry.id, Select(*is_null_backup, Expr<T>::Null(), min_max_backup));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3062 : : else
3063 [ # # # # : 0 : results.add(info.entry.id, min_max_backup.val());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3064 : :
3065 : : /*----- Move aggregate variables to access them later. ----*/
3066 [ # # # # : 0 : new (&agg_values[idx]) agg_t(std::make_pair(std::move(min_max), std::move(is_null)));
# # # # #
# # # # #
# # # # #
# # # #
# ]
3067 [ # # # # : 0 : new (&agg_value_backups[idx]) agg_backup_t(std::make_pair(
# # # # #
# # # # #
# # # # #
# # # #
# ]
3068 : : std::move(min_max_backup), std::move(is_null_backup)
3069 : : ));
3070 : 0 : };
3071 : 0 : auto &n = as<const Numeric>(*info.entry.type);
3072 [ # # # ]: 0 : switch (n.kind) {
3073 : : case Numeric::N_Int:
3074 : : case Numeric::N_Decimal:
3075 [ # # # # : 0 : switch (n.size()) {
# ]
3076 : 0 : default: M_unreachable("invalid size");
3077 : 0 : case 8: min_max.template operator()<int8_t >(); break;
3078 : 0 : case 16: min_max.template operator()<int16_t>(); break;
3079 : 0 : case 32: min_max.template operator()<int32_t>(); break;
3080 : 0 : case 64: min_max.template operator()<int64_t>(); break;
3081 : : }
3082 : 0 : break;
3083 : : case Numeric::N_Float:
3084 [ # # ]: 0 : if (n.size() <= 32)
3085 : 0 : min_max.template operator()<float>();
3086 : : else
3087 : 0 : min_max.template operator()<double>();
3088 : 0 : }
3089 : 0 : break;
3090 : : }
3091 : : case m::Function::FN_AVG: {
3092 : 0 : Var<Doublex1> avg;
3093 [ # # ]: 0 : Global<Doublex1> avg_backup(0.0); // initialize with neutral element 0
3094 : 0 : std::optional<Var<Boolx1>> is_null;
3095 : 0 : std::optional<Global<Boolx1>> is_null_backup;
3096 : :
3097 : : /*----- Set local aggregate variables to global backups. -----*/
3098 [ # # ]: 0 : avg = avg_backup;
3099 [ # # ]: 0 : if (nullable) {
3100 [ # # ]: 0 : is_null_backup.emplace(true); // AVG is initially NULL
3101 [ # # ]: 0 : is_null.emplace(*is_null_backup);
3102 : 0 : }
3103 : :
3104 : : /*----- Add global aggregate to result environment to access it in other function. -----*/
3105 [ # # ]: 0 : if (nullable)
3106 [ # # # # : 0 : results.add(info.entry.id, Select(*is_null_backup, _Doublex1::Null(), avg_backup));
# # # # ]
3107 : : else
3108 [ # # # # : 0 : results.add(info.entry.id, avg_backup.val());
# # # # ]
3109 : :
3110 : : /*----- Move aggregate variables to access them later. ----*/
3111 [ # # # # ]: 0 : new (&agg_values[idx]) agg_t(std::make_pair(std::move(avg), std::move(is_null)));
3112 [ # # # # ]: 0 : new (&agg_value_backups[idx]) agg_backup_t(std::make_pair(
3113 : : std::move(avg_backup), std::move(is_null_backup)
3114 : : ));
3115 : :
3116 : : break;
3117 : 0 : }
3118 : : case m::Function::FN_SUM: {
3119 : 0 : auto sum = [&]<typename T>() {
3120 : 0 : Var<PrimitiveExpr<T>> sum;
3121 [ # # # # : 0 : Global<PrimitiveExpr<T>> sum_backup(T(0)); // initialize with neutral element 0
# # # # #
# # # ]
3122 : 0 : std::optional<Var<Boolx1>> is_null;
3123 : 0 : std::optional<Global<Boolx1>> is_null_backup;
3124 : :
3125 : : /*----- Set local aggregate variables to global backups. -----*/
3126 [ # # # # : 0 : sum = sum_backup;
# # # # #
# # # ]
3127 [ # # # # : 0 : if (nullable) {
# # # # #
# # # ]
3128 [ # # # # : 0 : is_null_backup.emplace(true); // SUM is initially NULL
# # # # #
# # # ]
3129 [ # # # # : 0 : is_null.emplace(*is_null_backup);
# # # # #
# # # ]
3130 : 0 : }
3131 : :
3132 : : /*----- Add global aggregate to result environment to access it in other function. -----*/
3133 [ # # # # : 0 : if (nullable)
# # # # #
# # # ]
3134 [ # # # # : 0 : results.add(info.entry.id, Select(*is_null_backup, Expr<T>::Null(), sum_backup));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3135 : : else
3136 [ # # # # : 0 : results.add(info.entry.id, sum_backup.val());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3137 : :
3138 : : /*----- Move aggregate variables to access them later. ----*/
3139 [ # # # # : 0 : new (&agg_values[idx]) agg_t(std::make_pair(std::move(sum), std::move(is_null)));
# # # # #
# # # # #
# # # # #
# # # #
# ]
3140 [ # # # # : 0 : new (&agg_value_backups[idx]) agg_backup_t(std::make_pair(
# # # # #
# # # # #
# # # # #
# # # #
# ]
3141 : : std::move(sum_backup), std::move(is_null_backup)
3142 : : ));
3143 : 0 : };
3144 : 0 : auto &n = as<const Numeric>(*info.entry.type);
3145 [ # # # ]: 0 : switch (n.kind) {
3146 : : case Numeric::N_Int:
3147 : : case Numeric::N_Decimal:
3148 [ # # # # : 0 : switch (n.size()) {
# ]
3149 : 0 : default: M_unreachable("invalid size");
3150 : 0 : case 8: sum.template operator()<int8_t >(); break;
3151 : 0 : case 16: sum.template operator()<int16_t>(); break;
3152 : 0 : case 32: sum.template operator()<int32_t>(); break;
3153 : 0 : case 64: sum.template operator()<int64_t>(); break;
3154 : : }
3155 : 0 : break;
3156 : : case Numeric::N_Float:
3157 [ # # ]: 0 : if (n.size() <= 32)
3158 : 0 : sum.template operator()<float>();
3159 : : else
3160 : 0 : sum.template operator()<double>();
3161 : 0 : }
3162 : 0 : break;
3163 : : }
3164 : : case m::Function::FN_COUNT: {
3165 : 0 : Var<I64x1> count;
3166 [ # # ]: 0 : Global<I64x1> count_backup(0); // initialize with neutral element 0
3167 : : /* no `is_null` variables needed since COUNT will not be NULL */
3168 : :
3169 : : /*----- Set local aggregate variable to global backup. -----*/
3170 [ # # ]: 0 : count = count_backup;
3171 : :
3172 : : /*----- Add global aggregate to result environment to access it in other function. -----*/
3173 [ # # # # : 0 : results.add(info.entry.id, count_backup.val());
# # # # ]
3174 : :
3175 : : /*----- Move aggregate variables to access them later. ----*/
3176 [ # # ]: 0 : new (&agg_values[idx]) agg_t(std::move(count));
3177 [ # # ]: 0 : new (&agg_value_backups[idx]) agg_backup_t(std::move(count_backup));
3178 : :
3179 : : break;
3180 : 0 : }
3181 : : }
3182 : 0 : }
3183 : :
3184 : : /*----- Initialize keys and their backups. -----*/
3185 : 0 : auto init = [&]<typename T>(std::size_t idx) {
3186 : 0 : const bool nullable = M.grouping.schema()[idx].nullable();
3187 : :
3188 : 0 : Var<PrimitiveExpr<T>> key;
3189 [ # # # # : 0 : Global<PrimitiveExpr<T>> key_backup;
# # # # #
# # # #
# ]
3190 : 0 : std::optional<Var<Boolx1>> is_null;
3191 : 0 : std::optional<Global<Boolx1>> is_null_backup;
3192 : :
3193 : : /*----- Set local key variables to global backups. -----*/
3194 [ # # # # : 0 : key = key_backup;
# # # # #
# # # #
# ]
3195 [ # # # # : 0 : if (nullable) {
# # # # #
# # # #
# ]
3196 [ # # # # : 0 : is_null_backup.emplace();
# # # # #
# # # #
# ]
3197 [ # # # # : 0 : is_null.emplace(*is_null_backup);
# # # # #
# # # #
# ]
3198 : 0 : }
3199 : :
3200 : : try {
3201 [ # # # # : 0 : auto id = M.grouping.schema()[idx].id;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3202 [ # # # # : 0 : key_schema.find(id);
# # # # #
# # # #
# ]
3203 : :
3204 : : /*----- Add global key to result environment to access it in other function. -----*/
3205 [ # # # # : 0 : if (nullable)
# # # # #
# # # #
# ]
3206 [ # # # # : 0 : results.add(id, Select(*is_null_backup, Expr<T>::Null(), key_backup));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3207 : : else
3208 [ # # # # : 0 : results.add(id, key_backup.val());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3209 [ # # # # : 0 : } catch (invalid_argument&) {
# # # # #
# # # #
# ]
3210 : : /* skip adding to result environment for duplicate keys since they must not be used afterwards */
3211 [ # # # # : 0 : }
# # # # #
# # # #
# ]
3212 : :
3213 : : /*----- Move key variables to access them later. ----*/
3214 [ # # # # : 0 : new (&key_values[idx]) key_t(std::make_pair(std::move(key), std::move(is_null)));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3215 [ # # # # : 0 : new (&key_value_backups[idx]) key_backup_t(std::make_pair(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3216 : : std::move(key_backup), std::move(is_null_backup)
3217 : : ));
3218 : 0 : };
3219 [ # # ]: 0 : for (std::size_t idx = 0; idx < num_keys; ++idx) {
3220 : 0 : visit(overloaded{
3221 : 0 : [&](const Boolean&) { init.template operator()<bool>(idx); },
3222 : 0 : [&](const Numeric &n) {
3223 [ # # # ]: 0 : switch (n.kind) {
3224 : : case Numeric::N_Int:
3225 : : case Numeric::N_Decimal:
3226 [ # # # # : 0 : switch (n.size()) {
# ]
3227 : 0 : default: M_unreachable("invalid size");
3228 : 0 : case 8: init.template operator()<int8_t >(idx); break;
3229 : 0 : case 16: init.template operator()<int16_t>(idx); break;
3230 : 0 : case 32: init.template operator()<int32_t>(idx); break;
3231 : 0 : case 64: init.template operator()<int64_t>(idx); break;
3232 : : }
3233 : 0 : break;
3234 : : case Numeric::N_Float:
3235 [ # # ]: 0 : if (n.size() <= 32)
3236 : 0 : init.template operator()<float>(idx);
3237 : : else
3238 : 0 : init.template operator()<double>(idx);
3239 : 0 : }
3240 : 0 : },
3241 : 0 : [&](const CharacterSequence &cs) {
3242 : 0 : Var<Ptr<Charx1>> key;
3243 [ # # ]: 0 : Global<Ptr<Charx1>> key_backup;
3244 : : /* no `is_null` variables needed since pointer types must not be NULL */
3245 : :
3246 : : /*----- Set local key variable to global backup. -----*/
3247 [ # # ]: 0 : key = key_backup;
3248 : :
3249 : : try {
3250 [ # # # # ]: 0 : auto id = M.grouping.schema()[idx].id;
3251 [ # # ]: 0 : key_schema.find(id);
3252 : :
3253 : : /*----- Add global key to result environment to access it in other function. -----*/
3254 [ # # # # : 0 : NChar str(key_backup.val(), M.grouping.schema()[idx].nullable(), cs.length, cs.is_varying);
# # ]
3255 [ # # # # ]: 0 : results.add(id, std::move(str));
3256 [ # # ]: 0 : } catch (invalid_argument&) {
3257 : : /* skip adding to result environment for duplicate keys since they must not be used
3258 : : * afterwards */
3259 [ # # ]: 0 : }
3260 : :
3261 : : /*----- Move key variables to access them later. ----*/
3262 [ # # ]: 0 : new (&key_values[idx]) key_t(std::move(key));
3263 [ # # ]: 0 : new (&key_value_backups[idx]) key_backup_t(std::move(key_backup));
3264 : 0 : },
3265 : 0 : [&](const Date&) { init.template operator()<int32_t>(idx); },
3266 : 0 : [&](const DateTime&) { init.template operator()<int64_t>(idx); },
3267 : 0 : [](auto&&) { M_unreachable("invalid type"); },
3268 : 0 : }, *M.grouping.schema()[idx].type);
3269 : 0 : }
3270 : 0 : }),
3271 [ # # ]: 0 : /* pipeline= */ [&](){
3272 : 0 : auto &env = CodeGenContext::Get().env();
3273 : :
3274 : : /*----- If predication is used, introduce pred. var. and update it before computing aggregates. -----*/
3275 : 0 : std::optional<Var<Boolx1>> pred;
3276 [ # # ]: 0 : if (env.predicated()) {
3277 [ # # # # ]: 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
3278 [ # # # # : 0 : pred = env.extract_predicate<_Boolx1>().is_true_and_not_null();
# # ]
3279 : 0 : }
3280 : :
3281 : : /*----- Compute aggregates. -----*/
3282 [ # # ]: 0 : Block reset_aggs("ordered_grouping.reset_aggs", false),
3283 [ # # ]: 0 : update_aggs("ordered_grouping.update_aggs", false),
3284 [ # # ]: 0 : update_avg_aggs("ordered_grouping.update_avg_aggs", false);
3285 [ # # ]: 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
3286 : 0 : auto &info = aggregates[idx];
3287 : :
3288 : 0 : bool is_min = false; ///< flag to indicate whether aggregate function is MIN
3289 [ # # # # : 0 : switch (info.fnid) {
# # ]
3290 : : default:
3291 [ # # ]: 0 : M_unreachable("unsupported aggregate function");
3292 : : case m::Function::FN_MIN:
3293 : 0 : is_min = true; // set flag and delegate to MAX case
3294 : : case m::Function::FN_MAX: {
3295 [ # # ]: 0 : M_insist(info.args.size() == 1, "MIN and MAX aggregate functions expect exactly one argument");
3296 : 0 : const auto &arg = *info.args[0];
3297 : 0 : auto min_max = [&]<typename T>() {
3298 : 0 : auto neutral = is_min ? std::numeric_limits<T>::max()
3299 : 0 : : std::numeric_limits<T>::lowest();
3300 : :
3301 : 0 : auto &[min_max, is_null] = *M_notnull((
3302 : : std::get_if<std::pair<Var<PrimitiveExpr<T>>, std::optional<Var<Boolx1>>>>(&agg_values[idx])
3303 : : ));
3304 : :
3305 : 0 : BLOCK_OPEN(reset_aggs) {
3306 [ # # # # : 0 : min_max = neutral;
# # # # #
# # # ]
3307 [ # # # # : 0 : if (is_null)
# # # # #
# # # ]
3308 [ # # # # : 0 : is_null->set_true();
# # # # #
# # # ]
3309 : : }
3310 : :
3311 : 0 : BLOCK_OPEN(update_aggs) {
3312 [ # # # # : 0 : auto _arg = env.compile(arg);
# # # # #
# # # ]
3313 [ # # # # : 0 : Expr<T> _new_val = convert<Expr<T>>(_arg);
# # # # #
# # # ]
3314 [ # # # # : 0 : M_insist(_new_val.can_be_null() == bool(is_null));
# # # # #
# # # # #
# # # # #
# # # #
# ]
3315 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # ]
3316 : : M_insist_no_ternary_logic();
3317 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, Expr<T>::Null()) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3318 [ # # # # : 0 : auto [new_val_, new_val_is_null_] = _new_val_pred.split();
# # # # #
# # # ]
3319 [ # # # # : 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # ]
3320 : :
3321 : : if constexpr (std::floating_point<T>) {
3322 [ # # # # : 0 : min_max = Select(new_val_is_null,
# # # # ]
3323 : : min_max, // ignore NULL
3324 [ # # # # : 0 : is_min ? min(min_max, new_val_) // update old min with new value
# # # # ]
3325 [ # # # # ]: 0 : : max(min_max, new_val_)); // update old max with new value
3326 : : } else {
3327 [ # # # # : 0 : const Var<PrimitiveExpr<T>> new_val(new_val_); // due to multiple uses
# # # # ]
3328 [ # # # # : 0 : auto cmp = is_min ? new_val < min_max : new_val > min_max;
# # # # #
# # # # #
# # # # #
# # # #
# ]
3329 : : #if 1
3330 [ # # # # : 0 : min_max = Select(new_val_is_null,
# # # # #
# # # # #
# # ]
3331 : : min_max, // ignore NULL
3332 [ # # # # : 0 : Select(cmp,
# # # # ]
3333 : : new_val, // update to new value
3334 : : min_max)); // do not update
3335 : : #else
3336 : : IF (not new_val_is_null and cmp) {
3337 : : min_max = new_val;
3338 : : };
3339 : : #endif
3340 : 0 : }
3341 [ # # # # : 0 : *is_null = *is_null and new_val_is_null; // MIN/MAX is NULL iff all values are NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
3342 : 0 : } else {
3343 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, neutral) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3344 [ # # # # : 0 : auto new_val_ = _new_val_pred.insist_not_null();
# # # # #
# # # ]
3345 : : if constexpr (std::floating_point<T>) {
3346 [ # # # # : 0 : min_max = is_min ? min(min_max, new_val_) // update old min with new value
# # # # #
# # # ]
3347 [ # # # # ]: 0 : : max(min_max, new_val_); // update old max with new value
3348 : : } else {
3349 [ # # # # : 0 : const Var<PrimitiveExpr<T>> new_val(new_val_); // due to multiple uses
# # # # ]
3350 [ # # # # : 0 : auto cmp = is_min ? new_val < min_max : new_val > min_max;
# # # # #
# # # # #
# # # # #
# # # #
# ]
3351 : : #if 1
3352 [ # # # # : 0 : min_max = Select(cmp,
# # # # #
# # # # #
# # ]
3353 : : new_val, // update to new value
3354 : : min_max); // do not update
3355 : : #else
3356 : : IF (cmp) {
3357 : : min_max = new_val;
3358 : : };
3359 : : #endif
3360 : 0 : }
3361 : 0 : }
3362 : 0 : }
3363 : 0 : };
3364 [ # # ]: 0 : auto &n = as<const Numeric>(*info.entry.type);
3365 [ # # # ]: 0 : switch (n.kind) {
3366 : : case Numeric::N_Int:
3367 : : case Numeric::N_Decimal:
3368 [ # # # # : 0 : switch (n.size()) {
# # # ]
3369 [ # # ]: 0 : default: M_unreachable("invalid size");
3370 [ # # ]: 0 : case 8: min_max.template operator()<int8_t >(); break;
3371 [ # # ]: 0 : case 16: min_max.template operator()<int16_t>(); break;
3372 [ # # ]: 0 : case 32: min_max.template operator()<int32_t>(); break;
3373 [ # # ]: 0 : case 64: min_max.template operator()<int64_t>(); break;
3374 : : }
3375 : 0 : break;
3376 : : case Numeric::N_Float:
3377 [ # # # # ]: 0 : if (n.size() <= 32)
3378 [ # # ]: 0 : min_max.template operator()<float>();
3379 : : else
3380 [ # # ]: 0 : min_max.template operator()<double>();
3381 : 0 : }
3382 : 0 : break;
3383 : : }
3384 : : case m::Function::FN_AVG:
3385 : 0 : break; // skip here and handle later
3386 : : case m::Function::FN_SUM: {
3387 [ # # ]: 0 : M_insist(info.args.size() == 1, "SUM aggregate function expects exactly one argument");
3388 : 0 : const auto &arg = *info.args[0];
3389 : :
3390 : 0 : auto sum = [&]<typename T>() {
3391 : 0 : auto &[sum, is_null] = *M_notnull((
3392 : : std::get_if<std::pair<Var<PrimitiveExpr<T>>, std::optional<Var<Boolx1>>>>(&agg_values[idx])
3393 : : ));
3394 : :
3395 : 0 : BLOCK_OPEN(reset_aggs) {
3396 [ # # # # : 0 : sum = T(0);
# # # # #
# # # # #
# # # # #
# # # #
# ]
3397 [ # # # # : 0 : if (is_null)
# # # # #
# # # ]
3398 [ # # # # : 0 : is_null->set_true();
# # # # #
# # # ]
3399 : : }
3400 : :
3401 : 0 : BLOCK_OPEN(update_aggs) {
3402 [ # # # # : 0 : auto _arg = env.compile(arg);
# # # # #
# # # ]
3403 [ # # # # : 0 : Expr<T> _new_val = convert<Expr<T>>(_arg);
# # # # #
# # # ]
3404 [ # # # # : 0 : M_insist(_new_val.can_be_null() == bool(is_null));
# # # # #
# # # # #
# # # # #
# # # #
# ]
3405 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # ]
3406 : : M_insist_no_ternary_logic();
3407 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, Expr<T>::Null()) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3408 [ # # # # : 0 : auto [new_val, new_val_is_null_] = _new_val_pred.split();
# # # # #
# # # ]
3409 [ # # # # : 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # ]
3410 : :
3411 [ # # # # : 0 : sum += Select(new_val_is_null,
# # # # #
# # # # #
# # # # #
# # # #
# ]
3412 : 0 : T(0), // ignore NULL
3413 : : new_val); // add new value to old sum
3414 [ # # # # : 0 : *is_null = *is_null and new_val_is_null; // SUM is NULL iff all values are NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
3415 : 0 : } else {
3416 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, T(0)) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3417 [ # # # # : 0 : sum += _new_val_pred.insist_not_null(); // add new value to old sum
# # # # #
# # # # #
# # # # #
# # # #
# ]
3418 : 0 : }
3419 : 0 : }
3420 : 0 : };
3421 [ # # ]: 0 : auto &n = as<const Numeric>(*info.entry.type);
3422 [ # # # ]: 0 : switch (n.kind) {
3423 : : case Numeric::N_Int:
3424 : : case Numeric::N_Decimal:
3425 [ # # # # : 0 : switch (n.size()) {
# # # ]
3426 [ # # ]: 0 : default: M_unreachable("invalid size");
3427 [ # # ]: 0 : case 8: sum.template operator()<int8_t >(); break;
3428 [ # # ]: 0 : case 16: sum.template operator()<int16_t>(); break;
3429 [ # # ]: 0 : case 32: sum.template operator()<int32_t>(); break;
3430 [ # # ]: 0 : case 64: sum.template operator()<int64_t>(); break;
3431 : : }
3432 : 0 : break;
3433 : : case Numeric::N_Float:
3434 [ # # # # ]: 0 : if (n.size() <= 32)
3435 [ # # ]: 0 : sum.template operator()<float>();
3436 : : else
3437 [ # # ]: 0 : sum.template operator()<double>();
3438 : 0 : }
3439 : 0 : break;
3440 : : }
3441 : : case m::Function::FN_COUNT: {
3442 [ # # ]: 0 : M_insist(info.args.size() <= 1, "COUNT aggregate function expects at most one argument");
3443 [ # # # # : 0 : M_insist(info.entry.type->is_integral() and info.entry.type->size() == 64);
# # ]
3444 : :
3445 [ # # ]: 0 : auto &count = *M_notnull(std::get_if<Var<I64x1>>(&agg_values[idx]));
3446 : :
3447 [ # # ]: 0 : BLOCK_OPEN(reset_aggs) {
3448 [ # # ]: 0 : count = int64_t(0);
3449 : : }
3450 : :
3451 [ # # ]: 0 : BLOCK_OPEN(update_aggs) {
3452 [ # # ]: 0 : if (info.args.empty()) {
3453 [ # # # # : 0 : count += pred ? pred->to<int64_t>() : I64x1(1); // increment old count by 1 iff `pred` is true
# # # # ]
3454 : 0 : } else {
3455 [ # # ]: 0 : auto _new_val = env.compile(*info.args[0]);
3456 [ # # # # ]: 0 : if (can_be_null(_new_val)) {
3457 : : M_insist_no_ternary_logic();
3458 [ # # # # : 0 : I64x1 inc = pred ? (not_null(_new_val) and *pred).to<int64_t>()
# # # # #
# # # # #
# # # # #
# ]
3459 [ # # # # ]: 0 : : not_null(_new_val).to<int64_t>();
3460 [ # # ]: 0 : count += inc; // increment old count by 1 iff new value is present and `pred` is true
3461 : 0 : } else {
3462 [ # # ]: 0 : discard(_new_val); // since it is not needed in this case
3463 [ # # # # : 0 : I64x1 inc = pred ? pred->to<int64_t>() : I64x1(1);
# # ]
3464 [ # # ]: 0 : count += inc; // increment old count by 1 iff new value is present and `pred` is true
3465 : 0 : }
3466 : 0 : }
3467 : : }
3468 : 0 : break;
3469 : : }
3470 : : }
3471 : 0 : }
3472 : :
3473 : : /*----- Compute AVG aggregates after others to ensure that running count is already created. -----*/
3474 [ # # ]: 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
3475 : 0 : auto &info = aggregates[idx];
3476 : :
3477 [ # # ]: 0 : if (info.fnid == m::Function::FN_AVG) {
3478 [ # # ]: 0 : M_insist(info.args.size() == 1, "AVG aggregate function expects exactly one argument");
3479 : 0 : const auto &arg = *info.args[0];
3480 [ # # # # ]: 0 : M_insist(info.entry.type->is_double());
3481 : :
3482 [ # # ]: 0 : auto it = avg_aggregates.find(info.entry.id);
3483 [ # # ]: 0 : M_insist(it != avg_aggregates.end());
3484 : 0 : const auto &avg_info = it->second;
3485 [ # # ]: 0 : M_insist(avg_info.compute_running_avg,
3486 : : "AVG aggregate may only occur for running average computations");
3487 : :
3488 [ # # ]: 0 : auto &[avg, is_null] = *M_notnull((
3489 : : std::get_if<std::pair<Var<Doublex1>, std::optional<Var<Boolx1>>>>(&agg_values[idx])
3490 : : ));
3491 : :
3492 [ # # ]: 0 : BLOCK_OPEN(reset_aggs) {
3493 [ # # # # ]: 0 : avg = 0.0;
3494 [ # # ]: 0 : if (is_null)
3495 [ # # ]: 0 : is_null->set_true();
3496 : : }
3497 : :
3498 [ # # ]: 0 : BLOCK_OPEN(update_avg_aggs) {
3499 : : /* Compute AVG as iterative mean as described in Knuth, The Art of Computer Programming
3500 : : * Vol 2, section 4.2.2. */
3501 [ # # ]: 0 : auto running_count_idx = std::distance(
3502 : 0 : aggregates.cbegin(),
3503 [ # # ]: 0 : std::find_if(aggregates.cbegin(), aggregates.cend(), [&avg_info](const auto &info){
3504 : 0 : return info.entry.id == avg_info.running_count;
3505 : : })
3506 : : );
3507 [ # # # # ]: 0 : M_insist(0 <= running_count_idx and running_count_idx < aggregates.size());
3508 [ # # ]: 0 : auto &running_count = *M_notnull(std::get_if<Var<I64x1>>(&agg_values[running_count_idx]));
3509 : :
3510 [ # # ]: 0 : auto _arg = env.compile(arg);
3511 [ # # ]: 0 : _Doublex1 _new_val = convert<_Doublex1>(_arg);
3512 [ # # # # ]: 0 : M_insist(_new_val.can_be_null() == bool(is_null));
3513 [ # # ]: 0 : if (_new_val.can_be_null()) {
3514 : : M_insist_no_ternary_logic();
3515 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, _Doublex1::Null()) : _new_val;
# # # # #
# # # ]
3516 [ # # ]: 0 : auto [new_val, new_val_is_null_] = _new_val_pred.split();
3517 [ # # ]: 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
3518 : :
3519 [ # # ]: 0 : auto delta_absolute = new_val - avg;
3520 [ # # # # ]: 0 : auto delta_relative = delta_absolute / running_count.to<double>();
3521 : :
3522 [ # # # # ]: 0 : avg += Select(new_val_is_null,
3523 : 0 : 0.0, // ignore NULL
3524 : : delta_relative); // update old average with new value
3525 [ # # # # ]: 0 : *is_null = *is_null and new_val_is_null; // AVG is NULL iff all values are NULL
3526 : 0 : } else {
3527 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, avg) : _new_val;
# # # # ]
3528 [ # # # # ]: 0 : auto delta_absolute = _new_val_pred.insist_not_null() - avg;
3529 [ # # # # ]: 0 : auto delta_relative = delta_absolute / running_count.to<double>();
3530 : :
3531 [ # # ]: 0 : avg += delta_relative; // update old average with new value
3532 : 0 : }
3533 : 0 : }
3534 : 0 : }
3535 : 0 : }
3536 : :
3537 : : /*----- Compute whether new group starts and update key variables accordingly. -----*/
3538 : 0 : std::optional<Boolx1> group_differs;
3539 [ # # ]: 0 : Block update_keys("ordered_grouping.update_grouping_keys", false);
3540 [ # # ]: 0 : for (std::size_t idx = 0; idx < num_keys; ++idx) {
3541 [ # # ]: 0 : std::visit(overloaded {
3542 : 0 : [&]<typename T>(Expr<T> value) -> void {
3543 : 0 : auto &[key_val, key_is_null] = *M_notnull((
3544 : : std::get_if<std::pair<Var<PrimitiveExpr<T>>, std::optional<Var<Boolx1>>>>(&key_values[idx])
3545 : : ));
3546 : 0 : M_insist(value.can_be_null() == bool(key_is_null));
3547 : :
3548 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # #
# ]
3549 : : M_insist_no_ternary_logic();
3550 [ # # # # : 0 : auto [val, is_null] = value.clone().split();
# # # # #
# # # #
# ]
3551 [ # # # # : 0 : auto null_differs = is_null != *key_is_null;
# # # # #
# # # #
# ]
3552 [ # # # # : 0 : Boolx1 key_differs = null_differs or (not *key_is_null and val != key_val);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3553 [ # # # # : 0 : if (group_differs)
# # # # #
# # # #
# ]
3554 [ # # # # : 0 : group_differs.emplace(key_differs or *group_differs);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3555 : : else
3556 [ # # # # : 0 : group_differs.emplace(key_differs);
# # # # #
# # # #
# ]
3557 : :
3558 [ # # # # : 0 : BLOCK_OPEN(update_keys) {
# # # # #
# # # #
# ]
3559 [ # # # # : 0 : std::tie(key_val, key_is_null) = value.split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3560 : : }
3561 : 0 : } else {
3562 [ # # # # : 0 : Boolx1 key_differs = key_val != value.clone().insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3563 [ # # # # : 0 : if (group_differs)
# # # # #
# # # #
# ]
3564 [ # # # # : 0 : group_differs.emplace(key_differs or *group_differs);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3565 : : else
3566 [ # # # # : 0 : group_differs.emplace(key_differs);
# # # # #
# # # #
# ]
3567 : :
3568 [ # # # # : 0 : BLOCK_OPEN(update_keys) {
# # # # #
# # # #
# ]
3569 [ # # # # : 0 : key_val = value.insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3570 : : }
3571 : 0 : }
3572 : 0 : },
3573 : 0 : [&](NChar value) -> void {
3574 : 0 : auto &key = *M_notnull(std::get_if<Var<Ptr<Charx1>>>(&key_values[idx]));
3575 : :
3576 [ # # ]: 0 : auto [key_addr, key_is_nullptr] = key.val().split();
3577 [ # # # # : 0 : auto [addr, is_nullptr] = value.val().clone().split();
# # ]
3578 [ # # ]: 0 : auto addr_differs = strncmp(
3579 [ # # # # : 0 : /* left= */ NChar(addr, value.can_be_null(), value.length(),
# # ]
3580 : 0 : value.guarantees_terminating_nul()),
3581 [ # # # # : 0 : /* right= */ NChar(key_addr, value.can_be_null(), value.length(),
# # ]
3582 : 0 : value.guarantees_terminating_nul()),
3583 [ # # ]: 0 : /* len= */ U32x1(value.length()),
3584 : : /* op= */ NE
3585 : : );
3586 [ # # ]: 0 : auto [addr_differs_value, addr_differs_is_null] = addr_differs.split();
3587 [ # # ]: 0 : addr_differs_is_null.discard(); // use potentially-null value but it is overruled if it is NULL
3588 [ # # # # ]: 0 : auto nullptr_differs = is_nullptr != key_is_nullptr.clone();
3589 [ # # # # : 0 : Boolx1 key_differs = nullptr_differs or (not key_is_nullptr and addr_differs_value);
# # # # ]
3590 [ # # ]: 0 : if (group_differs)
3591 [ # # # # : 0 : group_differs.emplace(key_differs or *group_differs);
# # ]
3592 : : else
3593 [ # # ]: 0 : group_differs.emplace(key_differs);
3594 : :
3595 [ # # ]: 0 : BLOCK_OPEN(update_keys) {
3596 [ # # # # ]: 0 : key = value.val();
3597 : : }
3598 : 0 : },
3599 : 0 : [](auto) -> void { M_unreachable("SIMDfication currently not supported"); },
3600 : 0 : [](std::monostate) -> void { M_unreachable("invalid expression"); },
3601 [ # # ]: 0 : }, env.compile(M.grouping.group_by()[idx].first.get()));
3602 : 0 : }
3603 [ # # ]: 0 : M_insist(bool(group_differs));
3604 : :
3605 : : /*----- Resume pipeline with computed group iff new one starts and emit code to reset aggregates. ---*/
3606 [ # # ]: 0 : M_insist(bool(first_iteration));
3607 [ # # ]: 0 : Boolx1 cond = *first_iteration or *group_differs; // `group_differs` defaulted in first iteration but overruled anyway
3608 [ # # # # : 0 : IF (pred ? Select(*pred, cond, false) : cond) { // ignore entries for which predication predicate is not fulfilled
# # # # #
# ]
3609 [ # # # # ]: 0 : IF (not *first_iteration) {
3610 : 0 : store_locals_to_globals();
3611 : 0 : emit_group_and_resume_pipeline();
3612 : 0 : reset_aggs.attach_to_current();
3613 : 0 : };
3614 : 0 : update_keys.attach_to_current();
3615 : 0 : *first_iteration = false;
3616 : 0 : };
3617 : :
3618 : : /*----- Emit code to update aggregates. -----*/
3619 [ # # ]: 0 : update_aggs.attach_to_current();
3620 [ # # ]: 0 : update_avg_aggs.attach_to_current(); // after others to ensure that running count is incremented before
3621 : 0 : },
3622 [ # # # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){
3623 : 0 : store_locals_to_globals();
3624 : :
3625 : : /*----- Destroy created aggregate values and their backups. -----*/
3626 [ # # ]: 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
3627 : 0 : agg_values[idx].~agg_t();
3628 : 0 : agg_value_backups[idx].~agg_backup_t();
3629 : 0 : }
3630 : :
3631 : 0 : M_insist(bool(first_iteration));
3632 : 0 : first_iteration_backup = *first_iteration;
3633 : 0 : first_iteration.reset();
3634 : 0 : })
3635 : : );
3636 : :
3637 : : /*----- If input was not empty, emit last group tuple in the current environment and resume the pipeline. -----*/
3638 [ # # # # ]: 0 : IF (not first_iteration_backup) {
3639 : 0 : emit_group_and_resume_pipeline();
3640 : 0 : };
3641 : :
3642 : : /*----- Delayed definition of function to emit group and resume pipeline (since result environment is needed). ---*/
3643 [ # # ]: 0 : auto fn = emit_group_and_resume_pipeline.make_function(); // outside BLOCK_OPEN-macro to register as current function
3644 [ # # # # ]: 0 : BLOCK_OPEN(fn.body()) {
3645 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
3646 [ # # # # ]: 0 : auto &env = CodeGenContext::Get().env();
3647 : :
3648 : : /*----- Emit setup code *before* possibly introducing temporary boolean variables to not overwrite them. -----*/
3649 [ # # ]: 0 : setup();
3650 : :
3651 : : /*----- Add computed group tuple to current environment. ----*/
3652 [ # # # # : 0 : for (auto &e : M.grouping.schema().deduplicate()) {
# # # # #
# ]
3653 : : try {
3654 [ # # ]: 0 : key_schema.find(e.id);
3655 [ # # ]: 0 : } catch (invalid_argument&) {
3656 : : continue; // skip duplicated keys since they must not be used afterwards
3657 [ # # ]: 0 : }
3658 : :
3659 [ # # # # ]: 0 : if (auto it = avg_aggregates.find(e.id);
3660 [ # # ]: 0 : it != avg_aggregates.end() and not it->second.compute_running_avg)
3661 : : { // AVG aggregates which is not yet computed, divide computed sum with computed count
3662 : 0 : auto &avg_info = it->second;
3663 [ # # ]: 0 : auto sum = results.get(avg_info.sum);
3664 [ # # # # : 0 : auto count = results.get<_I64x1>(avg_info.running_count).insist_not_null().to<double>();
# # ]
3665 [ # # # # ]: 0 : auto avg = convert<_Doublex1>(sum) / count;
3666 [ # # # # ]: 0 : if (avg.can_be_null()) {
3667 [ # # ]: 0 : _Var<Doublex1> var(avg); // introduce variable s.t. uses only load from it
3668 [ # # # # : 0 : env.add(e.id, var);
# # ]
3669 : 0 : } else {
3670 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
3671 [ # # # # ]: 0 : Var<Doublex1> var(avg.insist_not_null());
3672 [ # # # # : 0 : env.add(e.id, _Doublex1(var));
# # # # ]
3673 : 0 : }
3674 : 0 : } else { // part of key or already computed aggregate
3675 [ # # ]: 0 : std::visit(overloaded {
3676 : 0 : [&]<typename T>(Expr<T> value) -> void {
3677 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # #
# ]
3678 : 0 : Var<Expr<T>> var(value); // introduce variable s.t. uses only load from it
3679 [ # # # # : 0 : env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3680 : 0 : } else {
3681 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
3682 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(value.insist_not_null());
# # # # #
# # # #
# ]
3683 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3684 : 0 : }
3685 : 0 : },
3686 : 0 : [&](NChar value) -> void {
3687 [ # # ]: 0 : Var<Ptr<Charx1>> var(value.val()); // introduce variable s.t. uses only load from it
3688 [ # # # # : 0 : env.add(e.id, NChar(var, value.can_be_null(), value.length(),
# # # # #
# ]
3689 : 0 : value.guarantees_terminating_nul()));
3690 : 0 : },
3691 : 0 : [](auto) -> void { M_unreachable("SIMDfication currently not supported"); },
3692 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
3693 [ # # ]: 0 : }, results.get(e.id)); // do not extract to be able to access for not-yet-computed AVG aggregates
3694 : : }
3695 : : }
3696 : :
3697 : : /*----- Resume pipeline. -----*/
3698 [ # # ]: 0 : pipeline();
3699 : :
3700 : : /*----- Emit teardown code. -----*/
3701 [ # # ]: 0 : teardown();
3702 : 0 : }
3703 : 0 : }
3704 : :
3705 : :
3706 : : /*======================================================================================================================
3707 : : * Aggregation
3708 : : *====================================================================================================================*/
3709 : :
3710 : 0 : ConditionSet Aggregation::pre_condition(std::size_t child_idx, const std::tuple<const AggregationOperator*>&)
3711 : : {
3712 : 0 : M_insist(child_idx == 0);
3713 : :
3714 : 0 : ConditionSet pre_cond;
3715 : :
3716 : 0 : return pre_cond;
3717 [ # # ]: 0 : }
3718 : :
3719 : 0 : ConditionSet Aggregation::post_condition(const Match<Aggregation> &M)
3720 : : {
3721 : 0 : ConditionSet post_cond;
3722 : :
3723 : : /*----- Aggregation does not introduce predication. -----*/
3724 [ # # # # ]: 0 : post_cond.add_condition(Predicated(false));
3725 : :
3726 : : /*----- Aggregation does implicitly sort the data since only one tuple is produced. -----*/
3727 : 0 : Sortedness::order_t orders;
3728 [ # # # # : 0 : for (auto &e : M.aggregation.schema().deduplicate())
# # # # #
# ]
3729 [ # # # # ]: 0 : orders.add(e.id, Sortedness::O_UNDEF);
3730 [ # # # # ]: 0 : post_cond.add_condition(Sortedness(std::move(orders)));
3731 : :
3732 : : /*----- Aggregation does not introduce SIMD since only one tuple is produced. -----*/
3733 [ # # # # ]: 0 : post_cond.add_condition(NoSIMD());
3734 : :
3735 : 0 : return post_cond;
3736 [ # # ]: 0 : }
3737 : :
3738 : 0 : void Aggregation::execute(const Match<Aggregation> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
3739 : : {
3740 : 0 : Environment results; ///< stores result tuple
3741 : : ///> code to construct aggregates from SIMD vectors; must be emitted *after* the child pipeline is executed
3742 : 0 : std::vector<std::function<void(void)>> finalize_aggregates;
3743 : :
3744 : : /*----- Compute information about aggregates, especially about AVG aggregates. -----*/
3745 [ # # # # : 0 : auto p = compute_aggregate_info(M.aggregation.aggregates(), M.aggregation.schema());
# # ]
3746 : 0 : const auto &aggregates = p.first;
3747 : 0 : const auto &avg_aggregates = p.second;
3748 : :
3749 : : /*----- Set minimal number of SIMD lanes preferred to get fully utilized SIMD vectors for the aggregate args. ----*/
3750 : 0 : uint64_t min_size_in_bytes = 16;
3751 [ # # # # ]: 0 : for (auto &fn : M.aggregation.aggregates()) {
3752 [ # # ]: 0 : for (auto &e : fn.get().args) {
3753 [ # # # # ]: 0 : visit(overloaded {
3754 : 0 : [](const m::ast::ErrorExpr&) -> void { M_unreachable("no errors at this stage"); },
3755 : 0 : [](const m::ast::Designator&) -> void { /* nothing to be done */ },
3756 : 0 : [](const m::ast::Constant&) -> void { /* nothing to be done */ },
3757 : 0 : [](const m::ast::QueryExpr&) -> void { /* nothing to be done */ },
3758 : 0 : [&min_size_in_bytes](const m::ast::FnApplicationExpr &fn) -> void {
3759 : 0 : M_insist(not fn.get_function().is_aggregate(), "aggregate arguments must not be aggregates");
3760 : 0 : min_size_in_bytes = std::min(min_size_in_bytes, (fn.type()->size() + 7) / 8);
3761 [ # # ]: 0 : if (min_size_in_bytes == 1)
3762 : 0 : throw visit_stop_recursion(); // abort recursion
3763 : 0 : },
3764 : 0 : [&min_size_in_bytes](auto &e) -> void { // i.e. for unary and binary expressions
3765 : 0 : min_size_in_bytes = std::min(min_size_in_bytes, (e.type()->size() + 7) / 8);
3766 [ # # # # ]: 0 : if (min_size_in_bytes == 1)
3767 : 0 : throw visit_stop_recursion(); // abort recursion
3768 : 0 : }
3769 : 0 : }, *e, m::tag<m::ast::ConstPreOrderExprVisitor>());
3770 : : }
3771 : : }
3772 [ # # # # ]: 0 : CodeGenContext::Get().update_num_simd_lanes_preferred(16 / min_size_in_bytes); // set own preference
3773 : :
3774 : : /*----- Set minimal number of SIMD lanes preferred to be able to compute running averages. ----*/
3775 [ # # # # ]: 0 : if (std::any_of(avg_aggregates.begin(), avg_aggregates.end(), [](auto &i){ return i.second.compute_running_avg; }))
3776 [ # # # # ]: 0 : CodeGenContext::Get().update_num_simd_lanes_preferred(4); // set own preference
3777 : :
3778 : : /*----- Create child function. -----*/
3779 [ # # # # : 0 : FUNCTION(aggregation_child_pipeline, void(void)) // create function for pipeline
# # # # ]
3780 : : {
3781 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
3782 : :
3783 : : #ifndef NDEBUG
3784 : : std::size_t num_simd_lanes; ///< to insist that setup, pipeline, and teardown callbacks use the same value
3785 : : #endif
3786 : : void *_agg_values; ///< *local* values of the computed aggregates
3787 : : void *_agg_value_backups; ///< *global* value backups of the computed aggregates
3788 : :
3789 [ # # ]: 0 : M.child->execute(
3790 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&]() {
3791 : 0 : auto execute_setup = [&]<std::size_t L>() {
3792 : : #ifndef NDEBUG
3793 : 0 : num_simd_lanes = L;
3794 : : #endif
3795 : :
3796 : : /*----- Initialize aggregates helper structures. -----*/
3797 : : using agg_t = agg_t_<false, L>;
3798 : : using agg_backup_t = agg_t_<true, L>;
3799 [ # # # # : 0 : auto agg_values = new agg_t[aggregates.size()];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3800 [ # # # # : 0 : auto agg_value_backups = new agg_backup_t[aggregates.size()];
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3801 : :
3802 : : /*----- Store aggregates helper structures for pipeline and teardown callbacks. -----*/
3803 : 0 : _agg_values = static_cast<void*>(agg_values);
3804 : 0 : _agg_value_backups = static_cast<void*>(agg_value_backups);
3805 : :
3806 : : /*----- Initialize aggregates and their backups. -----*/
3807 [ # # # # : 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
# # # # #
# # # ]
3808 : 0 : auto &info = aggregates[idx];
3809 : :
3810 : 0 : bool is_min = false; ///< flag to indicate whether aggregate function is MIN
3811 [ # # # # : 0 : switch (info.fnid) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3812 : : default:
3813 : 0 : M_unreachable("unsupported aggregate function");
3814 : : case m::Function::FN_MIN:
3815 : 0 : is_min = true; // set flag and delegate to MAX case
3816 : : case m::Function::FN_MAX: {
3817 : 0 : auto min_max = [&]<typename T>() {
3818 : 0 : auto neutral = is_min ? std::numeric_limits<T>::max()
3819 : 0 : : std::numeric_limits<T>::lowest();
3820 : :
3821 : 0 : Var<PrimitiveExpr<T, L>> min_max;
3822 [ # # # # : 0 : Global<PrimitiveExpr<T, L>> min_max_backup(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3823 : : neutral // initialize with neutral element +inf or -inf
3824 : : );
3825 [ # # # # : 0 : Var<Bool<L>> is_null;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3826 [ # # # # : 0 : Global<Bool<L>> is_null_backup(true); // MIN/MAX is initially NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3827 : :
3828 : : /*----- Set local aggregate variables to global backups. -----*/
3829 [ # # # # : 0 : min_max = min_max_backup;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3830 [ # # # # : 0 : is_null = is_null_backup;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3831 : :
3832 : : /*----- Add global aggregate to result env. to access it in other function. -----*/
3833 : : if constexpr (L == 1) { // scalar
3834 [ # # # # : 0 : PrimitiveExpr<T> value = min_max_backup;
# # # # #
# # # ]
3835 [ # # # # : 0 : Boolx1 is_null = is_null_backup;
# # # # #
# # # ]
3836 [ # # # # : 0 : results.add(info.entry.id, Select(is_null, Expr<T>::Null(), value));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3837 : 0 : } else { // vectorial
3838 : : /* Create lambda which emits the computation of the final *scalar* aggregate.
3839 : : * This can then be called in the pipeline function starting at the aggregation
3840 : : * operator s.t. the emitted variable is a local of the correct function.
3841 : : * Do not access the global variables inside the lambda using closure by
3842 : : * reference since they are already destroyed when the lambda will be called.
3843 : : * Instead, copy their values into the lambda. However, since DSL expressions
3844 : : * are not const-copy-constructible, we have to allocate them on the heap and
3845 : : * destroy them manually inside the lambda. */
3846 [ # # # # : 0 : auto simd_min_max = new PrimitiveExpr<T, L>(min_max_backup.val());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3847 [ # # # # : 0 : auto simd_is_null = new Bool<L>(is_null_backup.val());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3848 [ # # # # : 0 : finalize_aggregates.emplace_back([&, is_min, simd_min_max, simd_is_null]() {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3849 [ # # # # : 0 : PrimitiveExpr<T> value = [&]<std::size_t... Is>(std::index_sequence<Is...>) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3850 [ # # # # : 0 : Var<PrimitiveExpr<T>> res(simd_min_max->clone().template extract<0>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3851 : 0 : auto update = [&]<std::size_t I>(){
3852 : : if constexpr (requires (PrimitiveExpr<T> v) { min(v, v); max(v, v); }) {
3853 [ # # # # : 0 : res = is_min ? min(res, simd_min_max->clone().template extract<I>())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3854 [ # # # # : 0 : : max(res, simd_min_max->clone().template extract<I>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3855 : : } else {
3856 [ # # # # : 0 : const Var<PrimitiveExpr<T>> extracted(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3857 [ # # # # : 0 : simd_min_max->clone().template extract<I>()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3858 : : ); // due to multiple uses
3859 [ # # # # : 0 : auto cmp = is_min ? extracted < res : extracted > res;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3860 : : #if 1
3861 [ # # # # : 0 : res = Select(cmp, extracted, res);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3862 : : #else
3863 : : IF (cmp) {
3864 : : res = extracted;
3865 : : };
3866 : : #endif
3867 : 0 : }
3868 : 0 : };
3869 [ # # # # : 0 : (update.template operator()<Is + 1>(), ...);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3870 [ # # # # : 0 : return res;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3871 : 0 : }(std::make_index_sequence<L - 1>{});
3872 [ # # # # : 0 : simd_min_max->discard(); // since it was always cloned
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3873 [ # # # # : 0 : Boolx1 is_null = simd_is_null->all_true();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3874 [ # # # # : 0 : results.add(info.entry.id, Select(is_null, Expr<T>::Null(), value));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3875 [ # # # # : 0 : delete simd_min_max; // destroy heap-allocated variable
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3876 [ # # # # : 0 : delete simd_is_null; // destroy heap-allocated variable
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3877 : 0 : });
3878 : : }
3879 : :
3880 : : /*----- Move aggregate variables to access them later. ----*/
3881 [ # # # # : 0 : new (&agg_values[idx]) agg_t(std::make_pair(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3882 : : std::move(min_max), std::move(is_null))
3883 : : );
3884 [ # # # # : 0 : new (&agg_value_backups[idx]) agg_backup_t(std::make_pair(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3885 : : std::move(min_max_backup), std::move(is_null_backup)
3886 : : ));
3887 : 0 : };
3888 : 0 : auto &n = as<const Numeric>(*info.entry.type);
3889 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # ]
3890 : : case Numeric::N_Int:
3891 : : case Numeric::N_Decimal:
3892 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3893 : 0 : default: M_unreachable("invalid size");
3894 : 0 : case 8: min_max.template operator()<int8_t >(); break;
3895 : 0 : case 16: min_max.template operator()<int16_t>(); break;
3896 : 0 : case 32: min_max.template operator()<int32_t>(); break;
3897 : 0 : case 64: min_max.template operator()<int64_t>(); break;
3898 : : }
3899 : 0 : break;
3900 : : case Numeric::N_Float:
3901 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # ]
3902 : 0 : min_max.template operator()<float>();
3903 : : else
3904 : 0 : min_max.template operator()<double>();
3905 : 0 : }
3906 : 0 : break;
3907 : : }
3908 : : case m::Function::FN_AVG:
3909 : 0 : break; // skip here and handle later
3910 : : case m::Function::FN_SUM: {
3911 : 0 : auto sum = [&]<typename T>() {
3912 : 0 : Var<PrimitiveExpr<T, L>> sum;
3913 [ # # # # : 0 : Global<PrimitiveExpr<T, L>> sum_backup(T(0)); // initialize with neutral element 0
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3914 [ # # # # : 0 : Var<Bool<L>> is_null;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3915 [ # # # # : 0 : Global<Bool<L>> is_null_backup(true); // SUM is initially NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3916 : :
3917 : : /*----- Set local aggregate variables to global backups. -----*/
3918 [ # # # # : 0 : sum = sum_backup;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3919 [ # # # # : 0 : is_null = is_null_backup;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3920 : :
3921 : : /*----- Add global aggregate to result env. to access it in other function. -----*/
3922 : : if constexpr (L == 1) { // scalar
3923 [ # # # # : 0 : PrimitiveExpr<T> value = sum_backup;
# # # # #
# # # ]
3924 [ # # # # : 0 : Boolx1 is_null = is_null_backup;
# # # # #
# # # ]
3925 [ # # # # : 0 : results.add(info.entry.id, Select(is_null, Expr<T>::Null(), value));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3926 : 0 : } else { // vectorial
3927 [ # # # # : 0 : PrimitiveExpr<T> value = [&]<std::size_t... Is>(std::index_sequence<Is...>) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3928 [ # # # # : 0 : return (sum_backup.template extract<Is>() + ...);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3929 : 0 : }(std::make_index_sequence<L>{});
3930 [ # # # # : 0 : Boolx1 is_null = is_null_backup.all_true();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3931 [ # # # # : 0 : results.add(info.entry.id, Select(is_null, Expr<T>::Null(), value));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3932 : 0 : }
3933 : :
3934 : : /*----- Move aggregate variables to access them later. ----*/
3935 [ # # # # : 0 : new (&agg_values[idx]) agg_t(std::make_pair(std::move(sum), std::move(is_null)));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3936 [ # # # # : 0 : new (&agg_value_backups[idx]) agg_backup_t(std::make_pair(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3937 : : std::move(sum_backup), std::move(is_null_backup)
3938 : : ));
3939 : 0 : };
3940 : 0 : auto &n = as<const Numeric>(*info.entry.type);
3941 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # ]
3942 : : case Numeric::N_Int:
3943 : : case Numeric::N_Decimal:
3944 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3945 : 0 : default: M_unreachable("invalid size");
3946 : 0 : case 8: sum.template operator()<int8_t >(); break;
3947 : 0 : case 16: sum.template operator()<int16_t>(); break;
3948 : 0 : case 32: sum.template operator()<int32_t>(); break;
3949 : 0 : case 64: sum.template operator()<int64_t>(); break;
3950 : : }
3951 : 0 : break;
3952 : : case Numeric::N_Float:
3953 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # ]
3954 : 0 : sum.template operator()<float>();
3955 : : else
3956 : 0 : sum.template operator()<double>();
3957 : 0 : }
3958 : 0 : break;
3959 : : }
3960 : : case m::Function::FN_COUNT: {
3961 : 0 : Var<I64<L>> count;
3962 [ # # # # : 0 : Global<I64<L>> count_backup(0); // initialize with neutral element 0
# # # # #
# # # ]
3963 : : /* no `is_null` variables needed since COUNT will not be NULL */
3964 : :
3965 : : /*----- Set local aggregate variable to global backup. -----*/
3966 [ # # # # : 0 : count = count_backup;
# # # # #
# # # ]
3967 : :
3968 : : /*----- Add global aggregate to result env. to access it in other function. -----*/
3969 : : if constexpr (L == 1) { // scalar
3970 [ # # ]: 0 : I64x1 value = count_backup;
3971 [ # # # # : 0 : results.add(info.entry.id, value);
# # ]
3972 : 0 : } else { // vectorial
3973 [ # # # # : 0 : I64x1 value = [&]<std::size_t... Is>(std::index_sequence<Is...>) {
# # # # #
# ]
3974 [ # # # # : 0 : return (count_backup.template extract<Is>() + ...);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
3975 : 0 : }(std::make_index_sequence<L>{});
3976 [ # # # # : 0 : results.add(info.entry.id, value);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
3977 : 0 : }
3978 : :
3979 : : /*----- Move aggregate variables to access them later. ----*/
3980 [ # # # # : 0 : new (&agg_values[idx]) agg_t(std::move(count));
# # # # #
# # # ]
3981 [ # # # # : 0 : new (&agg_value_backups[idx]) agg_backup_t(std::move(count_backup));
# # # # #
# # # ]
3982 : :
3983 : : break;
3984 : 0 : }
3985 : : }
3986 : 0 : }
3987 : :
3988 : : /*----- Initialize AVG aggregates after others to ensure that running count is initialized before. */
3989 [ # # # # : 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
# # # # #
# # # ]
3990 : 0 : auto &info = aggregates[idx];
3991 : :
3992 [ # # # # : 0 : if (info.fnid == m::Function::FN_AVG) {
# # # # #
# # # ]
3993 : 0 : Var<Double<L>> avg;
3994 [ # # # # : 0 : Global<Double<L>> avg_backup(0.0); // initialize with neutral element 0
# # # # #
# # # ]
3995 [ # # # # : 0 : Var<Bool<L>> is_null;
# # # # #
# # # ]
3996 [ # # # # : 0 : Global<Bool<L>> is_null_backup(true); // AVG is initially NULL
# # # # #
# # # ]
3997 : :
3998 : : /*----- Set local aggregate variables to global backups. -----*/
3999 [ # # # # : 0 : avg = avg_backup;
# # # # #
# # # ]
4000 [ # # # # : 0 : is_null = is_null_backup;
# # # # #
# # # ]
4001 : :
4002 : : /*----- Add global aggregate to result env. to access it in other function. -----*/
4003 : : if constexpr (L == 1) { // scalar
4004 [ # # ]: 0 : Doublex1 value = avg_backup;
4005 [ # # ]: 0 : Boolx1 is_null = is_null_backup;
4006 [ # # # # : 0 : results.add(info.entry.id, Select(is_null, _Doublex1::Null(), value));
# # # # ]
4007 : 0 : } else { // vectorial
4008 : : /* Create lambda which emits the computation of the final *scalar* aggregate.
4009 : : * This can then be called in the pipeline function starting at the aggregation
4010 : : * operator s.t. the emitted variable is a local of the correct function.
4011 : : * Do not access the global variables inside the lambda using closure by
4012 : : * reference since they are already destroyed when the lambda will be called.
4013 : : * Instead, copy their values into the lambda. However, since DSL expressions
4014 : : * are not const-copy-constructible, we have to allocate them on the heap and
4015 : : * destroy them manually inside the lambda. */
4016 [ # # # # : 0 : auto simd_avg = new Double<L>(avg_backup.val());
# # # # #
# # # # #
# # # # #
# ]
4017 [ # # # # : 0 : auto simd_is_null = new Bool<L>(is_null_backup.val());
# # # # #
# # # # #
# # # # #
# ]
4018 [ # # # # : 0 : auto simd_running_count = new I64<L>([&](){
# # # # #
# # # # #
# # # # #
# ]
4019 : 0 : auto it = avg_aggregates.find(info.entry.id);
4020 : 0 : M_insist(it != avg_aggregates.end());
4021 : 0 : const auto &avg_info = it->second;
4022 : 0 : M_insist(avg_info.compute_running_avg,
4023 : : "AVG aggregate may only occur for running average computations");
4024 : :
4025 : 0 : auto running_count_idx = std::distance(
4026 : 0 : aggregates.cbegin(),
4027 : 0 : std::find_if(
4028 : 0 : aggregates.cbegin(), aggregates.cend(), [&avg_info](const auto &info){
4029 : 0 : return info.entry.id == avg_info.running_count;
4030 : : })
4031 : : );
4032 [ # # # # : 0 : M_insist(0 <= running_count_idx and running_count_idx < aggregates.size());
# # # # #
# ]
4033 : :
4034 : 0 : auto &running_count =
4035 : 0 : *M_notnull(std::get_if<Global<I64<L>>>(&agg_value_backups[running_count_idx]));
4036 : 0 : return running_count.val();
4037 : : }());
4038 [ # # # # : 0 : finalize_aggregates.emplace_back([&, simd_avg, simd_is_null, simd_running_count]() {
# # # # #
# ]
4039 : 0 : Doublex1 value = [&]<std::size_t... Is>(std::index_sequence<Is...>) {
4040 [ # # # # : 0 : I64x1 count = (simd_running_count->clone().template extract<Is>() + ...);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4041 [ # # # # : 0 : const Var<Double<L>> simd_sum([&](){
# # # # #
# # # # #
# # # # #
# ]
4042 : : if constexpr (L != 2) {
4043 [ # # # # : 0 : return *simd_avg * simd_running_count->template to<double>();
# # # # ]
4044 : : } else {
4045 : 0 : M_unreachable("conversion from `I64<2>` to `Double<2>` not supported");
4046 : : return Double<L>(0.0); // this line is never reached; return dummy value
4047 : : }
4048 : 0 : }());
4049 [ # # # # : 0 : return (simd_sum.template extract<Is>() + ...) / count.to<double>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4050 : 0 : }(std::make_index_sequence<L>{});
4051 [ # # # # : 0 : Boolx1 is_null = simd_is_null->all_true();
# # # # #
# ]
4052 [ # # # # : 0 : results.add(info.entry.id, Select(is_null, _Doublex1::Null(), value));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4053 [ # # # # : 0 : delete simd_avg; // destroy heap-allocated variable
# # # # #
# ]
4054 [ # # # # : 0 : delete simd_is_null; // destroy heap-allocated variable
# # # # #
# ]
4055 [ # # # # : 0 : delete simd_running_count; // destroy heap-allocated variable
# # # # #
# ]
4056 : 0 : });
4057 : : }
4058 : :
4059 : : /*----- Move aggregate variables to access them later. ----*/
4060 [ # # # # : 0 : new (&agg_values[idx]) agg_t(std::make_pair(std::move(avg), std::move(is_null)));
# # # # #
# # # # #
# # # # #
# # # #
# ]
4061 [ # # # # : 0 : new (&agg_value_backups[idx]) agg_backup_t(std::make_pair(
# # # # #
# # # # #
# # # # #
# # # #
# ]
4062 : : std::move(avg_backup), std::move(is_null_backup)
4063 : : ));
4064 : 0 : }
4065 : 0 : }
4066 : 0 : };
4067 [ # # # # : 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
# # # ]
4068 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
4069 : 0 : case 1: execute_setup.operator()<1>(); break;
4070 : 0 : case 2: execute_setup.operator()<2>(); break;
4071 : 0 : case 4: execute_setup.operator()<4>(); break;
4072 : 0 : case 8: execute_setup.operator()<8>(); break;
4073 : 0 : case 16: execute_setup.operator()<16>(); break;
4074 : 0 : case 32: execute_setup.operator()<32>(); break;
4075 : : }
4076 : 0 : }),
4077 [ # # ]: 0 : /* pipeline= */ [&](){
4078 : 0 : auto execute_pipeline = [&]<std::size_t L>(){
4079 : : #ifndef NDEBUG
4080 : 0 : M_insist(num_simd_lanes == L,
4081 : : "number of SIMD lanes in pipeline callback must match the one in setup callback");
4082 : : #endif
4083 : :
4084 : : /*----- Get aggregates helper structures. -----*/
4085 : : using agg_t = agg_t_<false, L>;
4086 : : using agg_backup_t = agg_t_<true, L>;
4087 : 0 : auto agg_values = static_cast<agg_t*>(_agg_values);
4088 : 0 : auto agg_value_backups = static_cast<agg_backup_t*>(_agg_value_backups);
4089 : :
4090 : 0 : auto &env = CodeGenContext::Get().env();
4091 : :
4092 : : /*----- If predication is used, introduce pred. var. and update it before computing aggregates. --*/
4093 : 0 : std::optional<Var<Bool<L>>> pred;
4094 [ # # # # : 0 : if (env.predicated()) {
# # # # #
# # # ]
4095 : : if constexpr (sql_boolean_type<_Bool<L>>)
4096 [ # # # # : 0 : pred = env.extract_predicate<_Bool<L>>().is_true_and_not_null();
# # # # #
# # # # #
# # # # ]
4097 : : else
4098 [ # # # # : 0 : M_unreachable("invalid number of SIMD lanes");
# # ]
4099 : 0 : }
4100 : :
4101 : : /*----- Compute aggregates (except AVG). -----*/
4102 [ # # # # : 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
# # # # #
# # # ]
4103 : 0 : auto &info = aggregates[idx];
4104 : :
4105 : 0 : bool is_min = false; ///< flag to indicate whether aggregate function is MIN
4106 [ # # # # : 0 : switch (info.fnid) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4107 : : default:
4108 [ # # # # : 0 : M_unreachable("unsupported aggregate function");
# # # # #
# # # ]
4109 : : case m::Function::FN_MIN:
4110 : 0 : is_min = true; // set flag and delegate to MAX case
4111 : : case m::Function::FN_MAX: {
4112 [ # # # # : 0 : M_insist(info.args.size() == 1,
# # # # #
# # # ]
4113 : : "MIN and MAX aggregate functions expect exactly one argument");
4114 : 0 : const auto &arg = *info.args[0];
4115 : : auto min_max = overloaded{
4116 : 0 : [&]<typename T>() requires sql_type<Expr<T, L>> {
4117 : 0 : auto &[min_max, is_null] = *M_notnull((
4118 : : std::get_if<
4119 : : std::pair<Var<PrimitiveExpr<T, L>>, Var<Bool<L>>>
4120 : : >(&agg_values[idx])
4121 : : ));
4122 : :
4123 : 0 : auto _arg = env.compile(arg);
4124 [ # # # # : 0 : Expr<T, L> _new_val = convert<Expr<T, L>>(_arg);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4125 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4126 : : M_insist_no_ternary_logic();
4127 : : auto _new_val_pred =
4128 [ # # # # : 0 : pred ? Select(*pred, _new_val, Expr<T, L>::Null()) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4129 [ # # # # : 0 : auto [new_val_, new_val_is_null_] = _new_val_pred.split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4130 [ # # # # : 0 : const Var<Bool<L>> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4131 : :
4132 : : if constexpr (requires (PrimitiveExpr<T, L> v) { min(v, v); max(v, v); }) {
4133 [ # # # # : 0 : min_max = Select(new_val_is_null,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4134 : : min_max, // ignore NULL
4135 [ # # # # : 0 : is_min ? min(min_max, new_val_) // update old min with new value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4136 [ # # # # : 0 : : max(min_max, new_val_)); // update old max with new value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4137 : : } else {
4138 [ # # # # : 0 : const Var<PrimitiveExpr<T, L>> new_val(new_val_); // due to multiple uses
# # # # #
# # # # #
# # # # ]
4139 [ # # # # : 0 : auto cmp = is_min ? new_val < min_max : new_val > min_max;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4140 : : #if 1
4141 [ # # # # : 0 : min_max = Select(new_val_is_null,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4142 : : min_max, // ignore NULL
4143 [ # # # # : 0 : Select(cmp,
# # # # #
# # # # #
# # # # ]
4144 : : new_val, // update to new value
4145 : : min_max)); // do not update
4146 : : #else
4147 : : IF (not new_val_is_null and cmp) {
4148 : : min_max = new_val;
4149 : : };
4150 : : #endif
4151 : 0 : }
4152 [ # # # # : 0 : is_null = is_null and new_val_is_null; // MIN/MAX is NULL iff all values are NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4153 : 0 : } else {
4154 : 0 : auto neutral = is_min ? std::numeric_limits<T>::max()
4155 : 0 : : std::numeric_limits<T>::lowest();
4156 : : auto _new_val_pred =
4157 [ # # # # : 0 : pred ? Select(*pred, _new_val, PrimitiveExpr<T, L>(neutral)) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4158 [ # # # # : 0 : auto new_val_ = _new_val_pred.insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4159 : : if constexpr (requires (PrimitiveExpr<T, L> v) { min(v, v); max(v, v); }) {
4160 [ # # # # : 0 : min_max = is_min ? min(min_max, new_val_) // update old min with new value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4161 [ # # # # : 0 : : max(min_max, new_val_); // update old max with new value
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4162 : : } else {
4163 [ # # # # : 0 : const Var<PrimitiveExpr<T, L>> new_val(new_val_); // due to multiple uses
# # # # #
# # # # #
# # # # ]
4164 [ # # # # : 0 : auto cmp = is_min ? new_val < min_max : new_val > min_max;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4165 : : #if 1
4166 [ # # # # : 0 : min_max = Select(cmp,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4167 : : new_val, // update to new value
4168 : : min_max); // do not update
4169 : : #else
4170 : : IF (cmp) {
4171 : : min_max = new_val;
4172 : : };
4173 : : #endif
4174 : 0 : }
4175 [ # # # # : 0 : is_null.set_false(); // at least one non-NULL value is consumed
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4176 : 0 : }
4177 : 0 : },
4178 : 0 : []<typename>() { M_unreachable("invalid type for given number of SIMD lanes"); }
4179 : : };
4180 [ # # # # : 0 : auto &n = as<const Numeric>(*info.entry.type);
# # # # #
# # # ]
4181 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # ]
4182 : : case Numeric::N_Int:
4183 : : case Numeric::N_Decimal:
4184 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4185 [ # # # # : 0 : default: M_unreachable("invalid size");
# # # # #
# # # ]
4186 [ # # # # : 0 : case 8: min_max.template operator()<int8_t >(); break;
# # # # #
# # # ]
4187 [ # # # # : 0 : case 16: min_max.template operator()<int16_t>(); break;
# # # # #
# # # ]
4188 [ # # # # : 0 : case 32: min_max.template operator()<int32_t>(); break;
# # # # #
# # # ]
4189 [ # # # # : 0 : case 64: min_max.template operator()<int64_t>(); break;
# # # # #
# # # ]
4190 : : }
4191 : 0 : break;
4192 : : case Numeric::N_Float:
4193 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # # #
# # # # #
# # # #
# ]
4194 [ # # # # : 0 : min_max.template operator()<float>();
# # # # #
# # # ]
4195 : : else
4196 [ # # # # : 0 : min_max.template operator()<double>();
# # # # #
# # # ]
4197 : 0 : }
4198 : 0 : break;
4199 : : }
4200 : : case m::Function::FN_AVG:
4201 : 0 : break; // skip here and handle later
4202 : : case m::Function::FN_SUM: {
4203 [ # # # # : 0 : M_insist(info.args.size() == 1, "SUM aggregate function expects exactly one argument");
# # # # #
# # # ]
4204 : 0 : const auto &arg = *info.args[0];
4205 : :
4206 : : auto sum = overloaded{
4207 : 0 : [&]<typename T>() requires sql_type<Expr<T, L>> {
4208 : 0 : auto &[sum, is_null] = *M_notnull((
4209 : : std::get_if<
4210 : : std::pair<Var<PrimitiveExpr<T, L>>, Var<Bool<L>>>
4211 : : >(&agg_values[idx])
4212 : : ));
4213 : :
4214 : 0 : auto _arg = env.compile(arg);
4215 [ # # # # : 0 : Expr<T, L> _new_val = convert<Expr<T, L>>(_arg);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4216 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4217 : : M_insist_no_ternary_logic();
4218 : : auto _new_val_pred =
4219 [ # # # # : 0 : pred ? Select(*pred, _new_val, Expr<T, L>::Null()) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4220 [ # # # # : 0 : auto [new_val, new_val_is_null_] = _new_val_pred.split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4221 [ # # # # : 0 : const Var<Bool<L>> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4222 : :
4223 [ # # # # : 0 : sum += Select(new_val_is_null,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4224 [ # # # # : 0 : PrimitiveExpr<T, L>(T(0)), // ignore NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4225 : : new_val); // add new value to old sum
4226 [ # # # # : 0 : is_null = is_null and new_val_is_null; // SUM is NULL iff all values are NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4227 : 0 : } else {
4228 : : auto _new_val_pred =
4229 [ # # # # : 0 : pred ? Select(*pred, _new_val, PrimitiveExpr<T, L>(T(0))) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4230 [ # # # # : 0 : sum += _new_val_pred.insist_not_null(); // add new value to old sum
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4231 [ # # # # : 0 : is_null.set_false(); // at least one non-NULL value is consumed
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4232 : 0 : }
4233 : 0 : },
4234 : 0 : []<typename>() { M_unreachable("invalid type for given number of SIMD lanes"); }
4235 : : };
4236 [ # # # # : 0 : auto &n = as<const Numeric>(*info.entry.type);
# # # # #
# # # ]
4237 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # ]
4238 : : case Numeric::N_Int:
4239 : : case Numeric::N_Decimal:
4240 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4241 [ # # # # : 0 : default: M_unreachable("invalid size");
# # # # #
# # # ]
4242 [ # # # # : 0 : case 8: sum.template operator()<int8_t >(); break;
# # # # #
# # # ]
4243 [ # # # # : 0 : case 16: sum.template operator()<int16_t>(); break;
# # # # #
# # # ]
4244 [ # # # # : 0 : case 32: sum.template operator()<int32_t>(); break;
# # # # #
# # # ]
4245 [ # # # # : 0 : case 64: sum.template operator()<int64_t>(); break;
# # # # #
# # # ]
4246 : : }
4247 : 0 : break;
4248 : : case Numeric::N_Float:
4249 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # # #
# # # # #
# # # #
# ]
4250 [ # # # # : 0 : sum.template operator()<float>();
# # # # #
# # # ]
4251 : : else
4252 [ # # # # : 0 : sum.template operator()<double>();
# # # # #
# # # ]
4253 : 0 : }
4254 : 0 : break;
4255 : : }
4256 : : case m::Function::FN_COUNT: {
4257 [ # # # # : 0 : M_insist(info.args.size() <= 1, "COUNT aggregate function expects at most one argument");
# # # # #
# # # ]
4258 [ # # # # : 0 : M_insist(info.entry.type->is_integral() and info.entry.type->size() == 64);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4259 : :
4260 [ # # # # : 0 : auto &count = *M_notnull(std::get_if<Var<I64<L>>>(&agg_values[idx]));
# # # # #
# # # ]
4261 : :
4262 [ # # # # : 0 : if (info.args.empty()) {
# # # # #
# # # ]
4263 [ # # # # : 0 : count += pred ? pred->template to<int64_t>() : I64<L>(1); // increment old count by 1 iff `pred` is true
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4264 : 0 : } else {
4265 [ # # # # : 0 : auto _new_val = env.compile(*info.args[0]);
# # # # #
# # # ]
4266 [ # # # # : 0 : if (can_be_null(_new_val)) {
# # # # #
# # # # #
# # # # #
# # # #
# ]
4267 : : M_insist_no_ternary_logic();
4268 [ # # # # : 0 : I64<L> inc = pred ? (not_null<L>(_new_val) and *pred).template to<int64_t>()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4269 [ # # # # : 0 : : not_null<L>(_new_val).template to<int64_t>();
# # # # #
# # # # #
# # # # #
# # # #
# ]
4270 [ # # # # : 0 : count += inc; // increment old count by 1 iff new value is present and `pred` is true
# # # # #
# # # ]
4271 : 0 : } else {
4272 [ # # # # : 0 : discard(_new_val); // since it is not needed in this case
# # # # #
# # # ]
4273 [ # # # # : 0 : I64<L> inc = pred ? pred->template to<int64_t>() : I64<L>(1);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4274 [ # # # # : 0 : count += inc; // increment old count by 1 iff new value is present and `pred` is true
# # # # #
# # # ]
4275 : 0 : }
4276 : 0 : }
4277 : 0 : break;
4278 : : }
4279 : : }
4280 : 0 : }
4281 : :
4282 : : /*----- Compute AVG aggregates after others to ensure that running count is incremented before. --*/
4283 [ # # # # : 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
# # # # #
# # # ]
4284 : 0 : auto &info = aggregates[idx];
4285 : :
4286 [ # # # # : 0 : if (info.fnid == m::Function::FN_AVG) {
# # # # #
# # # ]
4287 [ # # # # : 0 : M_insist(info.args.size() == 1, "AVG aggregate function expects exactly one argument");
# # # # #
# # # ]
4288 : 0 : const auto &arg = *info.args[0];
4289 [ # # # # : 0 : M_insist(info.entry.type->is_double());
# # # # #
# # # # #
# # # # #
# # # #
# ]
4290 : :
4291 [ # # # # : 0 : auto it = avg_aggregates.find(info.entry.id);
# # # # #
# # # ]
4292 [ # # # # : 0 : M_insist(it != avg_aggregates.end());
# # # # #
# # # ]
4293 : 0 : const auto &avg_info = it->second;
4294 [ # # # # : 0 : M_insist(avg_info.compute_running_avg,
# # # # #
# # # ]
4295 : : "AVG aggregate may only occur for running average computations");
4296 : :
4297 [ # # # # : 0 : auto &[avg, is_null] = *M_notnull((
# # # # #
# # # ]
4298 : : std::get_if<std::pair<Var<Double<L>>, Var<Bool<L>>>>(&agg_values[idx])
4299 : : ));
4300 : :
4301 : : /* Compute AVG as iterative mean as described in Knuth, The Art of Computer Programming
4302 : : * Vol 2, section 4.2.2. */
4303 [ # # # # : 0 : auto running_count_idx = std::distance(
# # # # #
# # # ]
4304 : 0 : aggregates.cbegin(),
4305 [ # # # # : 0 : std::find_if(aggregates.cbegin(), aggregates.cend(), [&avg_info](const auto &info){
# # # # #
# # # ]
4306 : 0 : return info.entry.id == avg_info.running_count;
4307 : : })
4308 : : );
4309 [ # # # # : 0 : M_insist(0 <= running_count_idx and running_count_idx < aggregates.size());
# # # # #
# # # # #
# # # # #
# # # #
# ]
4310 [ # # # # : 0 : Double<L> running_count = [&](){
# # # # #
# # # ]
4311 : 0 : auto &running_count =
4312 : 0 : *M_notnull(std::get_if<Var<I64<L>>>(&agg_values[running_count_idx]));
4313 : : if constexpr (L != 2) {
4314 : 0 : return running_count.template to<double>();
4315 : : } else {
4316 : 0 : M_unreachable("conversion from `I64<2>` to `Double<2>` not supported");
4317 : : return Double<L>(0.0); // this line is never reached; return dummy value
4318 : : }
4319 : : }();
4320 : :
4321 [ # # # # : 0 : auto _arg = env.compile(arg);
# # # # #
# # # ]
4322 [ # # # # : 0 : _Double<L> _new_val = convert<_Double<L>>(_arg);
# # # # #
# # # ]
4323 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # #
# ]
4324 : : M_insist_no_ternary_logic();
4325 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, _Double<L>::Null()) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4326 [ # # # # : 0 : auto [new_val, new_val_is_null_] = _new_val_pred.split();
# # # # #
# # # ]
4327 [ # # # # : 0 : const Var<Bool<L>> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # ]
4328 : :
4329 [ # # # # : 0 : auto delta_absolute = new_val - avg;
# # # # #
# # # ]
4330 [ # # # # : 0 : auto delta_relative = delta_absolute / running_count;
# # # # #
# # # # #
# # # # #
# # # #
# ]
4331 : :
4332 [ # # # # : 0 : avg += Select(new_val_is_null,
# # # # #
# # # # #
# # # # #
# # # #
# ]
4333 [ # # # # : 0 : Double<L>(0.0), // ignore NULL
# # # # #
# # # ]
4334 : : delta_relative); // update old average with new value
4335 [ # # # # : 0 : is_null = is_null and new_val_is_null; // AVG is NULL iff all values are NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
4336 : 0 : } else {
4337 [ # # # # : 0 : auto _new_val_pred = pred ? Select(*pred, _new_val, avg) : _new_val;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4338 [ # # # # : 0 : auto delta_absolute = _new_val_pred.insist_not_null() - avg;
# # # # #
# # # # #
# # # # #
# # # #
# ]
4339 [ # # # # : 0 : auto delta_relative = delta_absolute / running_count;
# # # # #
# # # # #
# # # # #
# # # #
# ]
4340 : :
4341 [ # # # # : 0 : avg += delta_relative; // update old average with new value
# # # # #
# # # ]
4342 [ # # # # : 0 : is_null.set_false(); // at least one non-NULL value is consumed
# # # # #
# # # ]
4343 : 0 : }
4344 : 0 : }
4345 : 0 : }
4346 : 0 : };
4347 [ # # # # : 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
# # # ]
4348 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
4349 : 0 : case 1: execute_pipeline.operator()<1>(); break;
4350 : 0 : case 2: execute_pipeline.operator()<2>(); break;
4351 : 0 : case 4: execute_pipeline.operator()<4>(); break;
4352 : 0 : case 8: execute_pipeline.operator()<8>(); break;
4353 : 0 : case 16: execute_pipeline.operator()<16>(); break;
4354 : 0 : case 32: execute_pipeline.operator()<32>(); break;
4355 : : }
4356 : 0 : },
4357 [ # # # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){
4358 : 0 : auto execute_teardown = [&]<std::size_t L>(){
4359 : : #ifndef NDEBUG
4360 : 0 : M_insist(num_simd_lanes == L,
4361 : : "number of SIMD lanes in teardown callback must match the one in setup callback");
4362 : : #endif
4363 : :
4364 : : /*----- Get aggregates helper structures. -----*/
4365 : : using agg_t = agg_t_<false, L>;
4366 : : using agg_backup_t = agg_t_<true, L>;
4367 : 0 : auto agg_values = static_cast<agg_t*>(_agg_values);
4368 : 0 : auto agg_value_backups = static_cast<agg_backup_t*>(_agg_value_backups);
4369 : :
4370 : : /*----- Store local aggregate values to globals to access them in other function. -----*/
4371 [ # # # # : 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
# # # # #
# # # ]
4372 : 0 : auto &info = aggregates[idx];
4373 : :
4374 : 0 : bool is_min = false; ///< flag to indicate whether aggregate function is MIN
4375 [ # # # # : 0 : switch (info.fnid) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4376 : : default:
4377 : 0 : M_unreachable("unsupported aggregate function");
4378 : : case m::Function::FN_MIN:
4379 : 0 : is_min = true; // set flag and delegate to MAX case
4380 : : case m::Function::FN_MAX: {
4381 : 0 : auto min_max = [&]<typename T>() {
4382 : 0 : auto &[min_max_backup, is_null_backup] = *M_notnull((
4383 : : std::get_if<
4384 : : std::pair<Global<PrimitiveExpr<T, L>>, Global<Bool<L>>>
4385 : : >(&agg_value_backups[idx])
4386 : : ));
4387 : 0 : std::tie(min_max_backup, is_null_backup) = *M_notnull((
4388 : : std::get_if<std::pair<Var<PrimitiveExpr<T, L>>, Var<Bool<L>>>>(&agg_values[idx])
4389 : : ));
4390 : 0 : };
4391 : 0 : auto &n = as<const Numeric>(*info.entry.type);
4392 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # ]
4393 : : case Numeric::N_Int:
4394 : : case Numeric::N_Decimal:
4395 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4396 : 0 : default: M_unreachable("invalid size");
4397 : 0 : case 8: min_max.template operator()<int8_t >(); break;
4398 : 0 : case 16: min_max.template operator()<int16_t>(); break;
4399 : 0 : case 32: min_max.template operator()<int32_t>(); break;
4400 : 0 : case 64: min_max.template operator()<int64_t>(); break;
4401 : : }
4402 : 0 : break;
4403 : : case Numeric::N_Float:
4404 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # ]
4405 : 0 : min_max.template operator()<float>();
4406 : : else
4407 : 0 : min_max.template operator()<double>();
4408 : 0 : }
4409 : 0 : break;
4410 : : }
4411 : : case m::Function::FN_AVG: {
4412 : 0 : auto &[avg_backup, is_null_backup] = *M_notnull((
4413 : : std::get_if<std::pair<Global<Double<L>>, Global<Bool<L>>>>(&agg_value_backups[idx])
4414 : : ));
4415 : 0 : std::tie(avg_backup, is_null_backup) = *M_notnull((
4416 : : std::get_if<std::pair<Var<Double<L>>, Var<Bool<L>>>>(&agg_values[idx])
4417 : : ));
4418 : :
4419 : 0 : break;
4420 : : }
4421 : : case m::Function::FN_SUM: {
4422 : 0 : M_insist(info.args.size() == 1, "SUM aggregate function expects exactly one argument");
4423 : 0 : const auto &arg = *info.args[0];
4424 : :
4425 : 0 : auto sum = [&]<typename T>() {
4426 : 0 : auto &[sum_backup, is_null_backup] = *M_notnull((
4427 : : std::get_if<
4428 : : std::pair<Global<PrimitiveExpr<T, L>>, Global<Bool<L>>>
4429 : : >(&agg_value_backups[idx])
4430 : : ));
4431 : 0 : std::tie(sum_backup, is_null_backup) = *M_notnull((
4432 : : std::get_if<std::pair<Var<PrimitiveExpr<T, L>>, Var<Bool<L>>>>(&agg_values[idx])
4433 : : ));
4434 : 0 : };
4435 : 0 : auto &n = as<const Numeric>(*info.entry.type);
4436 [ # # # # : 0 : switch (n.kind) {
# # # # #
# # # # #
# # # # ]
4437 : : case Numeric::N_Int:
4438 : : case Numeric::N_Decimal:
4439 [ # # # # : 0 : switch (n.size()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4440 : 0 : default: M_unreachable("invalid size");
4441 : 0 : case 8: sum.template operator()<int8_t >(); break;
4442 : 0 : case 16: sum.template operator()<int16_t>(); break;
4443 : 0 : case 32: sum.template operator()<int32_t>(); break;
4444 : 0 : case 64: sum.template operator()<int64_t>(); break;
4445 : : }
4446 : 0 : break;
4447 : : case Numeric::N_Float:
4448 [ # # # # : 0 : if (n.size() <= 32)
# # # # #
# # # ]
4449 : 0 : sum.template operator()<float>();
4450 : : else
4451 : 0 : sum.template operator()<double>();
4452 : 0 : }
4453 : 0 : break;
4454 : : }
4455 : : case m::Function::FN_COUNT: {
4456 : 0 : auto &count_backup = *M_notnull(std::get_if<Global<I64<L>>>(&agg_value_backups[idx]));
4457 : 0 : count_backup = *M_notnull(std::get_if<Var<I64<L>>>(&agg_values[idx]));
4458 : :
4459 : 0 : break;
4460 : : }
4461 : : }
4462 : 0 : }
4463 : :
4464 : : /*----- Destroy created aggregates and their backups. -----*/
4465 [ # # # # : 0 : for (std::size_t idx = 0; idx < aggregates.size(); ++idx) {
# # # # #
# # # ]
4466 : 0 : agg_values[idx].~agg_t();
4467 : 0 : agg_value_backups[idx].~agg_backup_t();
4468 : 0 : }
4469 : :
4470 : : /*----- Free aggregates helper structures. -----*/
4471 [ # # # # : 0 : delete[] agg_values;
# # # # #
# # # ]
4472 [ # # # # : 0 : delete[] agg_value_backups;
# # # # #
# # # ]
4473 : 0 : };
4474 [ # # # # : 0 : switch (CodeGenContext::Get().num_simd_lanes()) {
# # # ]
4475 : 0 : default: M_unreachable("unsupported number of SIMD lanes");
4476 : 0 : case 1: execute_teardown.operator()<1>(); break;
4477 : 0 : case 2: execute_teardown.operator()<2>(); break;
4478 : 0 : case 4: execute_teardown.operator()<4>(); break;
4479 : 0 : case 8: execute_teardown.operator()<8>(); break;
4480 : 0 : case 16: execute_teardown.operator()<16>(); break;
4481 : 0 : case 32: execute_teardown.operator()<32>(); break;
4482 : : }
4483 : 0 : })
4484 : : );
4485 : 0 : }
4486 [ # # ]: 0 : aggregation_child_pipeline(); // call child function
4487 : :
4488 : : /*----- Emit setup code *before* possibly introducing temporary boolean variables to not overwrite them. -----*/
4489 [ # # ]: 0 : setup();
4490 : :
4491 : : /*----- Emit code to finalize aggregate computations. -----*/
4492 [ # # ]: 0 : for (auto &fn : finalize_aggregates)
4493 [ # # ]: 0 : fn();
4494 : :
4495 : : /*----- Add computed aggregates tuple to current environment. ----*/
4496 [ # # # # ]: 0 : auto &env = CodeGenContext::Get().env();
4497 [ # # # # : 0 : for (auto &e : M.aggregation.schema().deduplicate()) {
# # # # #
# ]
4498 [ # # # # ]: 0 : if (auto it = avg_aggregates.find(e.id);
4499 [ # # ]: 0 : it != avg_aggregates.end() and not it->second.compute_running_avg)
4500 : : { // AVG aggregates which is not yet computed, divide computed sum with computed count
4501 : 0 : auto &avg_info = it->second;
4502 [ # # ]: 0 : auto sum = results.get(avg_info.sum);
4503 [ # # # # : 0 : auto count = results.get<_I64x1>(avg_info.running_count).insist_not_null().to<double>();
# # ]
4504 [ # # # # ]: 0 : auto avg = convert<_Doublex1>(sum) / count;
4505 [ # # # # ]: 0 : M_insist(avg.can_be_null());
4506 [ # # ]: 0 : _Var<Doublex1> var(avg); // introduce variable s.t. uses only load from it
4507 [ # # # # : 0 : env.add(e.id, var);
# # ]
4508 : 0 : } else { // already computed aggregate
4509 [ # # ]: 0 : std::visit(overloaded {
4510 : 0 : [&]<typename T>(Expr<T> value) -> void {
4511 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # #
# ]
4512 : 0 : Var<Expr<T>> var(value); // introduce variable s.t. uses only load from it
4513 [ # # # # : 0 : env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4514 : 0 : } else {
4515 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
4516 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(value.insist_not_null());
# # # # #
# # # #
# ]
4517 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4518 : 0 : }
4519 : 0 : },
4520 : 0 : [](auto) -> void { M_unreachable("only scalar and non-string values must occur"); },
4521 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
4522 [ # # ]: 0 : }, results.get(e.id)); // do not extract to be able to access for not-yet-computed AVG aggregates
4523 : : }
4524 : : }
4525 : :
4526 : : /*----- Resume pipeline. -----*/
4527 [ # # # # ]: 0 : CodeGenContext::Get().set_num_simd_lanes(1); // since only a single tuple is produced
4528 [ # # ]: 0 : pipeline();
4529 : :
4530 : : /*----- Emit teardown code. -----*/
4531 [ # # ]: 0 : teardown();
4532 : 0 : }
4533 : :
4534 : :
4535 : : /*======================================================================================================================
4536 : : * Sorting
4537 : : *====================================================================================================================*/
4538 : :
4539 : : template<bool CmpPredicated>
4540 : 0 : ConditionSet Quicksort<CmpPredicated>::pre_condition(std::size_t child_idx, const std::tuple<const SortingOperator*>&)
4541 : : {
4542 : 0 : M_insist(child_idx == 0);
4543 : :
4544 : 0 : ConditionSet pre_cond;
4545 : :
4546 : : /*----- Sorting does not support SIMD. -----*/
4547 [ # # # # : 0 : pre_cond.add_condition(NoSIMD());
# # # # ]
4548 : :
4549 : 0 : return pre_cond;
4550 [ # # # # ]: 0 : }
4551 : :
4552 : : template<bool CmpPredicated>
4553 : 0 : ConditionSet Quicksort<CmpPredicated>::post_condition(const Match<Quicksort> &M)
4554 : : {
4555 : 0 : ConditionSet post_cond;
4556 : :
4557 : : /*----- Quicksort does not introduce predication. -----*/
4558 [ # # # # : 0 : post_cond.add_condition(Predicated(false));
# # # # ]
4559 : :
4560 : : /*----- Quicksort does sort the data. -----*/
4561 : 0 : Sortedness::order_t orders;
4562 [ # # # # : 0 : for (auto &o : M.sorting.order_by()) {
# # # # ]
4563 [ # # # # ]: 0 : Schema::Identifier id(o.first);
4564 [ # # # # : 0 : if (orders.find(id) == orders.cend())
# # # # #
# # # ]
4565 [ # # # # : 0 : orders.add(std::move(id), o.second ? Sortedness::O_ASC : Sortedness::O_DESC);
# # # # ]
4566 : 0 : }
4567 [ # # # # : 0 : post_cond.add_condition(Sortedness(std::move(orders)));
# # # # ]
4568 : :
4569 : : /*----- Sorting does not introduce SIMD. -----*/
4570 [ # # # # : 0 : post_cond.add_condition(NoSIMD());
# # # # ]
4571 : :
4572 : 0 : return post_cond;
4573 [ # # # # ]: 0 : }
4574 : :
4575 : : template<bool CmpPredicated>
4576 : 0 : void Quicksort<CmpPredicated>::execute(const Match<Quicksort> &M, setup_t setup, pipeline_t pipeline,
4577 : : teardown_t teardown)
4578 : : {
4579 : : /*----- Create infinite buffer to materialize the current results but resume the pipeline later. -----*/
4580 : 0 : M_insist(bool(M.materializing_factory), "`wasm::Quicksort` must have a factory for the materialized child");
4581 [ # # # # ]: 0 : const auto buffer_schema = M.child->get_matched_root().schema().drop_constants().deduplicate();
4582 [ # # # # : 0 : const auto sorting_schema = M.sorting.schema().drop_constants().deduplicate();
# # # # #
# # # ]
4583 [ # # # # ]: 0 : GlobalBuffer buffer(
4584 : 0 : buffer_schema, *M.materializing_factory, false, 0, std::move(setup), std::move(pipeline), std::move(teardown)
4585 : : );
4586 : :
4587 : : /*----- Create child function. -----*/
4588 [ # # # # : 0 : FUNCTION(sorting_child_pipeline, void(void)) // create function for pipeline
# # # # #
# # # # #
# # ]
4589 : : {
4590 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # ]
4591 : :
4592 [ # # # # ]: 0 : M.child->execute(
4593 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ buffer.setup(); }),
4594 : 0 : /* pipeline= */ [&](){ buffer.consume(); },
4595 [ # # # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ buffer.teardown(); })
4596 : : );
4597 : 0 : }
4598 [ # # # # ]: 0 : sorting_child_pipeline(); // call child function
4599 : :
4600 : : /*----- Invoke quicksort algorithm with buffer to sort. -----*/
4601 [ # # # # : 0 : quicksort<CmpPredicated>(buffer, M.sorting.order_by());
# # # # ]
4602 : :
4603 : : /*----- Process sorted buffer. -----*/
4604 [ # # # # ]: 0 : buffer.resume_pipeline(sorting_schema);
4605 : 0 : }
4606 : :
4607 : 0 : ConditionSet NoOpSorting::pre_condition(std::size_t child_idx,
4608 : : const std::tuple<const SortingOperator*> &partial_inner_nodes)
4609 : : {
4610 : 0 : M_insist(child_idx == 0);
4611 : :
4612 : 0 : ConditionSet pre_cond;
4613 : :
4614 : : /*----- NoOpSorting, i.e. a noop to match sorting, needs the data already sorted. -----*/
4615 : 0 : Sortedness::order_t orders;
4616 [ # # # # ]: 0 : for (auto &o : std::get<0>(partial_inner_nodes)->order_by()) {
4617 [ # # ]: 0 : Schema::Identifier id(o.first);
4618 [ # # # # : 0 : if (orders.find(id) == orders.cend())
# # ]
4619 [ # # # # ]: 0 : orders.add(std::move(id), o.second ? Sortedness::O_ASC : Sortedness::O_DESC);
4620 : 0 : }
4621 [ # # # # ]: 0 : pre_cond.add_condition(Sortedness(std::move(orders)));
4622 : :
4623 : 0 : return pre_cond;
4624 [ # # ]: 0 : }
4625 : :
4626 : 0 : void NoOpSorting::execute(const Match<NoOpSorting> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
4627 : : {
4628 [ # # ]: 0 : M.child->execute(std::move(setup), std::move(pipeline), std::move(teardown));
4629 : 0 : }
4630 : :
4631 : :
4632 : : /*======================================================================================================================
4633 : : * Join
4634 : : *====================================================================================================================*/
4635 : :
4636 : : template<bool Predicated>
4637 : 0 : ConditionSet NestedLoopsJoin<Predicated>::pre_condition(std::size_t, const std::tuple<const JoinOperator*>&)
4638 : : {
4639 : 0 : ConditionSet pre_cond;
4640 : :
4641 : : /*----- Nested-loops join does not support SIMD. -----*/
4642 [ # # # # : 0 : pre_cond.add_condition(NoSIMD());
# # # # ]
4643 : :
4644 : 0 : return pre_cond;
4645 [ # # # # ]: 0 : }
4646 : :
4647 : : template<bool Predicated>
4648 : 0 : ConditionSet NestedLoopsJoin<Predicated>::adapt_post_conditions(
4649 : : const Match<NestedLoopsJoin>&,
4650 : : std::vector<std::reference_wrapper<const ConditionSet>> &&post_cond_children)
4651 : : {
4652 : 0 : M_insist(post_cond_children.size() >= 2);
4653 : :
4654 : 0 : ConditionSet post_cond(post_cond_children.back().get()); // preserve conditions of right-most child
4655 : :
4656 : : if constexpr (Predicated) {
4657 : : /*----- Predicated nested-loops join introduces predication. -----*/
4658 [ # # # # ]: 0 : post_cond.add_or_replace_condition(m::Predicated(true));
4659 : : }
4660 : :
4661 : 0 : return post_cond;
4662 [ # # # # ]: 0 : }
4663 : :
4664 : : template<bool Predicated>
4665 : 0 : double NestedLoopsJoin<Predicated>::cost(const Match<NestedLoopsJoin> &M)
4666 : : {
4667 : 0 : double cost = 1;
4668 [ # # # # ]: 0 : for (auto &child : M.children)
4669 : 0 : cost *= child->get_matched_root().info().estimated_cardinality;
4670 : 0 : return cost;
4671 : : }
4672 : :
4673 : : template<bool Predicated>
4674 : 0 : void NestedLoopsJoin<Predicated>::execute(const Match<NestedLoopsJoin> &M, setup_t setup, pipeline_t pipeline,
4675 : : teardown_t teardown)
4676 : : {
4677 : 0 : const auto num_left_children = M.children.size() - 1; // all children but right-most one
4678 : :
4679 : 0 : std::vector<Schema> schemas; // to own adapted schemas
4680 [ # # # # ]: 0 : schemas.reserve(num_left_children);
4681 : 0 : std::vector<GlobalBuffer> buffers;
4682 [ # # # # ]: 0 : buffers.reserve(num_left_children);
4683 : :
4684 : : /*----- Process all but right-most child. -----*/
4685 [ # # # # ]: 0 : for (std::size_t i = 0; i < num_left_children; ++i) {
4686 : : /*----- Create function for each child. -----*/
4687 [ # # # # : 0 : FUNCTION(nested_loop_join_child_pipeline, void(void))
# # # # #
# # # # #
# # ]
4688 : : {
4689 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # ]
4690 : :
4691 : : /*----- Create infinite buffer to materialize the current results. -----*/
4692 [ # # # # ]: 0 : M_insist(bool(M.materializing_factories_[i]),
4693 : : "`wasm::NestedLoopsJoin` must have a factory for each materialized child");
4694 [ # # # # ]: 0 : const auto &schema = schemas.emplace_back(
4695 [ # # # # : 0 : M.children[i]->get_matched_root().schema().drop_constants().deduplicate()
# # # # #
# # # # #
# # ]
4696 : : );
4697 [ # # # # ]: 0 : if (i == 0) {
4698 : : /*----- Exactly one child (here left-most one) checks join predicate and resumes pipeline. -----*/
4699 [ # # # # ]: 0 : buffers.emplace_back(
4700 : 0 : /* schema= */ schema,
4701 : 0 : /* factory= */ *M.materializing_factories_[i],
4702 : 0 : /* load_simdfied= */ false,
4703 : 0 : /* num_tuples= */ 0, // i.e. infinite
4704 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent(),
4705 : 0 : /* pipeline= */ [&, pipeline=std::move(pipeline)](){
4706 : : if constexpr (Predicated) {
4707 : 0 : CodeGenContext::Get().env().add_predicate(M.join.predicate());
4708 : 0 : pipeline();
4709 : : } else {
4710 : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
4711 [ # # # # ]: 0 : IF (CodeGenContext::Get().env().compile<_Boolx1>(M.join.predicate()).is_true_and_not_null()) {
4712 : 0 : pipeline();
4713 : 0 : };
4714 : : }
4715 : 0 : },
4716 [ # # # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent()
4717 : : );
4718 : 0 : } else {
4719 : : /*----- All but exactly one child (here left-most one) load lastly inserted buffer again. -----*/
4720 : : /* All buffers are "connected" with each other by setting the pipeline callback as calling the
4721 : : * `resume_pipeline_inline()` method of the lastly inserted buffer. Therefore, calling
4722 : : * `resume_pipeline_inline()` on the lastly inserted buffer will load one tuple from it, recursively
4723 : : * call `resume_pipeline_inline()` on the buffer created before that which again loads one tuple from
4724 : : * it, and so on until the buffer inserted first (here the one of the left-most child) will load one
4725 : : * of its tuples and check the join predicate for this one cartesian-product-combination of result
4726 : : * tuples. */
4727 [ # # # # ]: 0 : buffers.emplace_back(
4728 : 0 : /* schema= */ schema,
4729 : 0 : /* factory= */ *M.materializing_factories_[i],
4730 : 0 : /* load_simdfied= */ false,
4731 : 0 : /* num_tuples= */ 0, // i.e. infinite
4732 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent(),
4733 : 0 : /* pipeline= */ [&](){ buffers.back().resume_pipeline_inline(); },
4734 [ # # # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent()
4735 : : );
4736 : : }
4737 : :
4738 : : /*----- Materialize the current result tuple in pipeline. -----*/
4739 [ # # # # ]: 0 : M.children[i]->execute(
4740 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ buffers.back().setup(); }),
4741 : 0 : /* pipeline= */ [&](){ buffers.back().consume(); },
4742 [ # # # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ buffers.back().teardown(); })
4743 : : );
4744 : 0 : }
4745 [ # # # # ]: 0 : nested_loop_join_child_pipeline(); // call child function
4746 : 0 : }
4747 : :
4748 : : /*----- Process right-most child. -----*/
4749 [ # # # # : 0 : M.children.back()->execute(
# # # # ]
4750 : 0 : /* setup= */ std::move(setup),
4751 : 0 : /* pipeline= */ [&](){ buffers.back().resume_pipeline_inline(); },
4752 : 0 : /* teardown= */ std::move(teardown)
4753 : : );
4754 : 0 : }
4755 : :
4756 : : template<bool UniqueBuild, bool Predicated>
4757 : 0 : ConditionSet SimpleHashJoin<UniqueBuild, Predicated>::pre_condition(
4758 : : std::size_t,
4759 : : const std::tuple<const JoinOperator*, const Wildcard*, const Wildcard*> &partial_inner_nodes)
4760 : : {
4761 : 0 : ConditionSet pre_cond;
4762 : :
4763 : : /*----- Simple hash join can only be used for binary joins on equi-predicates. -----*/
4764 : 0 : auto &join = *std::get<0>(partial_inner_nodes);
4765 [ # # # # : 0 : if (not join.predicate().is_equi())
# # # # #
# # # # #
# # # # #
# # # #
# ]
4766 [ # # # # : 0 : return ConditionSet::Make_Unsatisfiable();
# # # # ]
4767 : :
4768 : : if constexpr (UniqueBuild) {
4769 : : /*----- Decompose each clause of the join predicate of the form `A.x = B.y` into parts `A.x` and `B.y`. -----*/
4770 : 0 : auto &build = *std::get<1>(partial_inner_nodes);
4771 [ # # # # : 0 : for (auto &clause : join.predicate()) {
# # # # ]
4772 [ # # # # ]: 0 : M_insist(clause.size() == 1, "invalid equi-predicate");
4773 : 0 : auto &literal = clause[0];
4774 [ # # # # : 0 : auto &binary = as<const BinaryExpr>(literal.expr());
# # # # ]
4775 [ # # # # : 0 : M_insist((not literal.negative() and binary.tok == TK_EQUAL) or
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4776 : : (literal.negative() and binary.tok == TK_BANG_EQUAL), "invalid equi-predicate");
4777 [ # # # # : 0 : M_insist(is<const Designator>(binary.lhs), "invalid equi-predicate");
# # # # ]
4778 [ # # # # : 0 : M_insist(is<const Designator>(binary.rhs), "invalid equi-predicate");
# # # # ]
4779 [ # # # # : 0 : Schema::Identifier id_first(*binary.lhs), id_second(*binary.rhs);
# # # # ]
4780 [ # # # # : 0 : const auto &entry_build = build.schema().has(id_first) ? build.schema()[id_first].second
# # # # #
# # # # #
# # # # #
# ]
4781 [ # # # # : 0 : : build.schema()[id_second].second;
# # # # ]
4782 : :
4783 : : /*----- Unique simple hash join can only be used on unique build key. -----*/
4784 [ # # # # : 0 : if (not entry_build.unique())
# # # # ]
4785 [ # # # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
4786 [ # # # # ]: 0 : }
4787 : : }
4788 : :
4789 : : /*----- Simple hash join does not support SIMD. -----*/
4790 [ # # # # : 0 : pre_cond.add_condition(NoSIMD());
# # # # #
# # # # #
# # ]
4791 : :
4792 : 0 : return pre_cond;
4793 : 0 : }
4794 : :
4795 : : template<bool UniqueBuild, bool Predicated>
4796 : 0 : ConditionSet SimpleHashJoin<UniqueBuild, Predicated>::adapt_post_conditions(
4797 : : const Match<SimpleHashJoin>&,
4798 : : std::vector<std::reference_wrapper<const ConditionSet>> &&post_cond_children)
4799 : : {
4800 : 0 : M_insist(post_cond_children.size() == 2);
4801 : :
4802 : 0 : ConditionSet post_cond(post_cond_children[1].get()); // preserve conditions of right child
4803 : :
4804 : : if constexpr (Predicated) {
4805 : : /*----- Predicated simple hash join introduces predication. -----*/
4806 [ # # # # : 0 : post_cond.add_or_replace_condition(m::Predicated(true));
# # # # ]
4807 : : } else {
4808 : : /*----- Branching simple hash join does not introduce predication (it is already handled by the hash table). -*/
4809 [ # # # # : 0 : post_cond.add_or_replace_condition(m::Predicated(false));
# # # # ]
4810 : : }
4811 : :
4812 : 0 : return post_cond;
4813 [ # # # # : 0 : }
# # # # ]
4814 : :
4815 : : template<bool UniqueBuild, bool Predicated>
4816 : 0 : double SimpleHashJoin<UniqueBuild, Predicated>::cost(const Match<SimpleHashJoin> &M)
4817 : : {
4818 [ # # # # : 0 : if (options::simple_hash_join_ordering_strategy == option_configs::OrderingStrategy::BUILD_ON_LEFT)
# # # # ]
4819 : 0 : return (M.build.id() == M.children[0]->get_matched_root().id() ? 1.0 : 2.0) + (UniqueBuild ? 0.0 : 0.1);
4820 [ # # # # : 0 : else if (options::simple_hash_join_ordering_strategy == option_configs::OrderingStrategy::BUILD_ON_RIGHT)
# # # # ]
4821 : 0 : return M.build.id() == M.children[1]->get_matched_root().id() ? 1.0 : 2.0 + (UniqueBuild ? 0.0 : 0.1);
4822 : : else
4823 : 0 : return 1.5 * M.build.info().estimated_cardinality +
4824 : 0 : (UniqueBuild ? 1.0 : 1.1) * M.probe.info().estimated_cardinality;
4825 : 0 : }
4826 : :
4827 : : template<bool UniqueBuild, bool Predicated>
4828 : 0 : void SimpleHashJoin<UniqueBuild, Predicated>::execute(const Match<SimpleHashJoin> &M, setup_t setup,
4829 : : pipeline_t pipeline, teardown_t teardown)
4830 : : {
4831 : : // TODO: determine setup
4832 : 0 : const uint64_t PAYLOAD_SIZE_THRESHOLD_IN_BITS =
4833 : 0 : M.use_in_place_values ? std::numeric_limits<uint64_t>::max() : 0;
4834 : :
4835 [ # # # # : 0 : M_insist(((M.join.schema() | M.join.predicate().get_required()) & M.build.schema()) == M.build.schema());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4836 [ # # # # : 0 : M_insist(M.build.schema().drop_constants() == M.build.schema());
# # # # #
# # # # #
# # # # #
# # # #
# ]
4837 : 0 : const auto ht_schema = M.build.schema().deduplicate();
4838 : :
4839 : : /*----- Decompose each clause of the join predicate of the form `A.x = B.y` into parts `A.x` and `B.y`. -----*/
4840 [ # # # # : 0 : const auto [build_keys, probe_keys] = decompose_equi_predicate(M.join.predicate(), ht_schema);
# # # # #
# # # # #
# # ]
4841 : :
4842 : : /*----- Compute payload IDs and its total size in bits (ignoring padding). -----*/
4843 : 0 : std::vector<Schema::Identifier> payload_ids;
4844 : 0 : uint64_t payload_size_in_bits = 0;
4845 [ # # # # : 0 : for (auto &e : ht_schema) {
# # # # #
# # # # #
# # # # #
# # # #
# ]
4846 [ # # # # : 0 : if (not contains(build_keys, e.id)) {
# # # # #
# # # # #
# # ]
4847 [ # # # # : 0 : payload_ids.push_back(e.id);
# # # # ]
4848 [ # # # # : 0 : payload_size_in_bits += e.type->size();
# # # # ]
4849 : 0 : }
4850 : : }
4851 : :
4852 : : /*----- Compute initial capacity of hash table. -----*/
4853 [ # # # # : 0 : uint32_t initial_capacity = compute_initial_ht_capacity(M.build, M.load_factor);
# # # # ]
4854 : :
4855 : : /*----- Create hash table for build child. -----*/
4856 : 0 : std::unique_ptr<HashTable> ht;
4857 : 0 : std::vector<HashTable::index_t> build_key_indices;
4858 [ # # # # : 0 : for (auto &build_key : build_keys)
# # # # ]
4859 [ # # # # : 0 : build_key_indices.push_back(ht_schema[build_key].first);
# # # # #
# # # # #
# # ]
4860 [ # # # # : 0 : if (M.use_open_addressing_hashing) {
# # # # ]
4861 [ # # # # : 0 : if (payload_size_in_bits < PAYLOAD_SIZE_THRESHOLD_IN_BITS)
# # # # ]
4862 [ # # # # : 0 : ht = std::make_unique<GlobalOpenAddressingInPlaceHashTable>(ht_schema, std::move(build_key_indices),
# # # # ]
4863 : : initial_capacity);
4864 : : else
4865 [ # # # # : 0 : ht = std::make_unique<GlobalOpenAddressingOutOfPlaceHashTable>(ht_schema, std::move(build_key_indices),
# # # # ]
4866 : : initial_capacity);
4867 [ # # # # : 0 : if (M.use_quadratic_probing)
# # # # ]
4868 [ # # # # : 0 : as<OpenAddressingHashTableBase>(*ht).set_probing_strategy<QuadraticProbing>();
# # # # #
# # # # #
# # ]
4869 : : else
4870 [ # # # # : 0 : as<OpenAddressingHashTableBase>(*ht).set_probing_strategy<LinearProbing>();
# # # # #
# # # # #
# # ]
4871 : 0 : } else {
4872 [ # # # # : 0 : ht = std::make_unique<GlobalChainedHashTable>(ht_schema, std::move(build_key_indices), initial_capacity);
# # # # ]
4873 : : }
4874 : :
4875 : : /*----- Create function for build child. -----*/
4876 [ # # # # : 0 : FUNCTION(simple_hash_join_child_pipeline, void(void)) // create function for pipeline
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4877 : : {
4878 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # #
# # # # #
# # ]
4879 : :
4880 [ # # # # : 0 : M.children[0]->execute(
# # # # ]
4881 [ # # # # : 0 : /* setup= */ setup_t::Make_Without_Parent([&](){
# # # # ]
4882 : 0 : ht->setup();
4883 : 0 : ht->set_high_watermark(M.load_factor);
4884 : 0 : }),
4885 [ # # # # : 0 : /* pipeline= */ [&](){
# # # # #
# # # # #
# # ]
4886 : 0 : auto &env = CodeGenContext::Get().env();
4887 : :
4888 : 0 : std::optional<Boolx1> build_key_not_null;
4889 [ # # # # : 0 : for (auto &build_key : build_keys) {
# # # # ]
4890 [ # # # # : 0 : auto val = env.get(build_key);
# # # # ]
4891 [ # # # # : 0 : if (build_key_not_null)
# # # # ]
4892 [ # # # # : 0 : build_key_not_null.emplace(*build_key_not_null and not_null(val));
# # # # #
# # # # #
# # # # #
# # # #
# ]
4893 : : else
4894 [ # # # # : 0 : build_key_not_null.emplace(not_null(val));
# # # # #
# # # # #
# # ]
4895 : 0 : }
4896 [ # # # # : 0 : M_insist(bool(build_key_not_null));
# # # # ]
4897 [ # # # # : 0 : IF (*build_key_not_null) { // TODO: predicated version
# # # # #
# # # # #
# # ]
4898 : : /*----- Insert key. -----*/
4899 : 0 : std::vector<SQL_t> key;
4900 [ # # # # : 0 : for (auto &build_key : build_keys)
# # # # ]
4901 [ # # # # : 0 : key.emplace_back(env.get(build_key));
# # # # #
# # # # #
# # ]
4902 [ # # # # : 0 : auto entry = ht->emplace(std::move(key));
# # # # ]
4903 : :
4904 : : /*----- Insert payload. -----*/
4905 [ # # # # : 0 : for (auto &id : payload_ids) {
# # # # ]
4906 [ # # # # : 0 : std::visit(overloaded {
# # # # ]
4907 [ # # # # : 0 : [&]<sql_type T>(HashTable::reference_t<T> &&r) -> void { r = env.extract<T>(id); },
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4908 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
4909 [ # # # # : 0 : }, entry.extract(id));
# # # # ]
4910 : : }
4911 : 0 : };
4912 : 0 : },
4913 [ # # # # : 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ ht->teardown(); })
# # # # ]
4914 : : );
4915 : 0 : }
4916 [ # # # # : 0 : simple_hash_join_child_pipeline(); // call child function
# # # # ]
4917 : :
4918 [ # # # # : 0 : M.children[1]->execute(
# # # # ]
4919 [ # # # # : 0 : /* setup= */ setup_t(std::move(setup), [&](){ ht->setup(); }),
# # # # ]
4920 [ # # # # : 0 : /* pipeline= */ [&, pipeline=std::move(pipeline)](){
# # # # #
# # # # #
# # # # #
# # # #
# ]
4921 : 0 : auto &env = CodeGenContext::Get().env();
4922 : :
4923 : 0 : auto emit_tuple_and_resume_pipeline = [&, pipeline=std::move(pipeline)](HashTable::const_entry_t entry){
4924 : : /*----- Add found entry from hash table, i.e. from build child, to current environment. -----*/
4925 [ # # # # : 0 : for (auto &e : ht_schema) {
# # # # ]
4926 [ # # # # : 0 : if (not entry.has(e.id)) { // entry may not contain build key in case `ht->find()` was used
# # # # ]
4927 : 0 : M_insist(contains(build_keys, e.id));
4928 : 0 : M_insist(env.has(e.id), "build key must already be contained in the current environment");
4929 : 0 : continue;
4930 : : }
4931 : :
4932 [ # # # # : 0 : std::visit(overloaded {
# # # # #
# # # # #
# # ]
4933 : 0 : [&]<typename T>(HashTable::const_reference_t<Expr<T>> &&r) -> void {
4934 : 0 : Expr<T> value = r;
4935 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4936 [ # # # # : 0 : Var<Expr<T>> var(value); // introduce variable s.t. uses only load from it
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4937 [ # # # # : 0 : env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4938 : 0 : } else {
4939 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
4940 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(value.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4941 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4942 : 0 : }
4943 : 0 : },
4944 : 0 : [&](HashTable::const_reference_t<NChar> &&r) -> void {
4945 : 0 : NChar value(r);
4946 [ # # # # : 0 : Var<Ptr<Charx1>> var(value.val()); // introduce variable s.t. uses only load from it
# # # # #
# # # # #
# # ]
4947 [ # # # # : 0 : env.add(e.id, NChar(var, value.can_be_null(), value.length(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
4948 : 0 : value.guarantees_terminating_nul()));
4949 : 0 : },
4950 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
4951 : 0 : }, entry.extract(e.id));
4952 : : }
4953 : :
4954 : : /*----- Resume pipeline. -----*/
4955 : 0 : pipeline();
4956 : 0 : };
4957 : :
4958 : : /* TODO: may check for NULL on probe keys as well, branching + predicated version */
4959 : : /*----- Probe with probe key. -----*/
4960 : 0 : std::vector<SQL_t> key;
4961 [ # # # # : 0 : for (auto &probe_key : probe_keys)
# # # # ]
4962 [ # # # # : 0 : key.emplace_back(env.get(probe_key));
# # # # #
# # # # #
# # ]
4963 : : if constexpr (UniqueBuild) {
4964 : : /*----- Add build key to current environment since `ht->find()` will only return the payload values. -----*/
4965 [ # # # # ]: 0 : for (auto build_it = build_keys.cbegin(), probe_it = probe_keys.cbegin(); build_it != build_keys.cend();
4966 : 0 : ++build_it, ++probe_it)
4967 : : {
4968 [ # # # # ]: 0 : M_insist(probe_it != probe_keys.cend());
4969 [ # # # # : 0 : if (not env.has(*build_it)) // skip duplicated build keys and only add first occurrence
# # # # ]
4970 [ # # # # : 0 : env.add(*build_it, env.get(*probe_it)); // since build and probe keys match for join partners
# # # # #
# # # ]
4971 : 0 : }
4972 : :
4973 : : /*----- Try to find the *single* possible join partner. -----*/
4974 [ # # # # ]: 0 : auto p = ht->find(std::move(key));
4975 : 0 : auto &entry = p.first;
4976 : 0 : auto &found = p.second;
4977 : : if constexpr (Predicated) {
4978 [ # # # # ]: 0 : env.add_predicate(found);
4979 [ # # # # ]: 0 : emit_tuple_and_resume_pipeline(std::move(entry));
4980 : : } else {
4981 [ # # ]: 0 : IF (found) {
4982 [ # # ]: 0 : emit_tuple_and_resume_pipeline(std::move(entry));
4983 : 0 : };
4984 : : }
4985 : 0 : } else {
4986 : : /*----- Search for *all* join partners. -----*/
4987 [ # # # # : 0 : ht->for_each_in_equal_range(std::move(key), std::move(emit_tuple_and_resume_pipeline), Predicated);
# # # # ]
4988 : : }
4989 : 0 : },
4990 [ # # # # : 0 : /* teardown= */ teardown_t(std::move(teardown), [&](){ ht->teardown(); })
# # # # ]
4991 : : );
4992 : 0 : }
4993 : :
4994 : : template<bool SortLeft, bool SortRight, bool Predicated, bool CmpPredicated>
4995 : 0 : ConditionSet SortMergeJoin<SortLeft, SortRight, Predicated, CmpPredicated>::pre_condition(
4996 : : std::size_t child_idx,
4997 : : const std::tuple<const JoinOperator*, const Wildcard*, const Wildcard*> &partial_inner_nodes)
4998 : : {
4999 : 0 : ConditionSet pre_cond;
5000 : :
5001 : : /*----- Sort merge join can only be used for binary joins on conjunctions of equi-predicates. -----*/
5002 : 0 : auto &join = *std::get<0>(partial_inner_nodes);
5003 [ # # # # : 0 : if (not join.predicate().is_equi())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5004 [ # # # # : 0 : return ConditionSet::Make_Unsatisfiable();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5005 : :
5006 : : /*----- Decompose each clause of the join predicate of the form `A.x = B.y` into parts `A.x` and `B.y`. -----*/
5007 : 0 : auto parent = std::get<1>(partial_inner_nodes);
5008 : 0 : auto child = std::get<2>(partial_inner_nodes);
5009 [ # # # # : 0 : M_insist(parent);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5010 [ # # # # : 0 : M_insist(child_idx != 1 or child);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5011 : 0 : std::vector<Schema::Identifier> keys_parent, keys_child;
5012 [ # # # # : 0 : for (auto &clause : join.predicate()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5013 [ # # # # : 0 : M_insist(clause.size() == 1, "invalid equi-predicate");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5014 : 0 : auto &literal = clause[0];
5015 [ # # # # : 0 : auto &binary = as<const BinaryExpr>(literal.expr());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5016 [ # # # # : 0 : M_insist((not literal.negative() and binary.tok == TK_EQUAL) or
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5017 : : (literal.negative() and binary.tok == TK_BANG_EQUAL), "invalid equi-predicate");
5018 [ # # # # : 0 : M_insist(is<const Designator>(binary.lhs), "invalid equi-predicate");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5019 [ # # # # : 0 : M_insist(is<const Designator>(binary.rhs), "invalid equi-predicate");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5020 [ # # # # : 0 : Schema::Identifier id_first(*binary.lhs), id_second(*binary.rhs);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5021 [ # # # # : 0 : auto dummy = Schema::entry_type::CreateArtificial(); ///< dummy entry used in case of `child_idx` != 1, i.e. `child` is not yet set
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5022 [ # # # # : 0 : const auto &[entry_parent, entry_child] = parent->schema().has(id_first)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5023 [ # # # # : 0 : ? std::make_pair(parent->schema()[id_first].second, child_idx == 1 ? child->schema()[id_second].second : std::move(dummy))
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5024 [ # # # # : 0 : : std::make_pair(parent->schema()[id_second].second, child_idx == 1 ? child->schema()[id_first].second : std::move(dummy));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5025 [ # # # # : 0 : keys_parent.push_back(entry_parent.id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5026 [ # # # # : 0 : keys_child.push_back(entry_child.id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5027 : :
5028 : : /*----- Sort merge join can only be used on unique parent key. -----*/
5029 [ # # # # : 0 : if (not entry_parent.unique())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5030 [ # # # # : 0 : return ConditionSet::Make_Unsatisfiable();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5031 [ # # # # : 0 : }
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5032 [ # # # # : 0 : M_insist(keys_parent.size() == keys_child.size(), "number of found IDs differ");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5033 [ # # # # : 0 : M_insist(not keys_parent.empty(), "must find at least one ID");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5034 : :
5035 : : if constexpr (not SortLeft or not SortRight) {
5036 : : /*----- Sort merge join without sorting needs its data sorted on the respective key. -----*/
5037 : 0 : Sortedness::order_t orders;
5038 [ # # # # : 0 : M_insist(child_idx < 2);
# # # # #
# # # # #
# # # # #
# # # #
# ]
5039 [ # # # # : 0 : if (not SortLeft and child_idx == 0) {
# # # # #
# # # # #
# # ]
5040 [ # # # # : 0 : for (auto &key_parent : keys_parent) {
# # # # #
# # # # #
# # ]
5041 [ # # # # : 0 : if (orders.find(key_parent) == orders.cend())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5042 [ # # # # : 0 : orders.add(key_parent, Sortedness::O_ASC); // TODO: support different order
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5043 : : }
5044 [ # # # # : 0 : } else if (not SortRight and child_idx == 1) {
# # # # #
# # # # #
# # ]
5045 [ # # # # : 0 : for (auto &key_child : keys_child) {
# # # # #
# # # # #
# # ]
5046 [ # # # # : 0 : if (orders.find(key_child) == orders.cend())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5047 [ # # # # : 0 : orders.add(key_child, Sortedness::O_ASC); // TODO: support different order
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5048 : : }
5049 : 0 : }
5050 [ # # # # : 0 : pre_cond.add_condition(Sortedness(std::move(orders)));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5051 : 0 : }
5052 : :
5053 : : /*----- Sort merge join does not support SIMD. -----*/
5054 [ # # # # : 0 : pre_cond.add_condition(NoSIMD());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5055 : :
5056 : 0 : return pre_cond;
5057 : 0 : }
5058 : :
5059 : : template<bool SortLeft, bool SortRight, bool Predicated, bool CmpPredicated>
5060 : 0 : ConditionSet SortMergeJoin<SortLeft, SortRight, Predicated, CmpPredicated>::adapt_post_conditions(
5061 : : const Match<SortMergeJoin> &M,
5062 : : std::vector<std::reference_wrapper<const ConditionSet>> &&post_cond_children)
5063 : : {
5064 : 0 : M_insist(post_cond_children.size() == 2);
5065 : :
5066 : 0 : ConditionSet post_cond;
5067 : :
5068 : : if constexpr (Predicated) {
5069 : : /*----- Predicated sort merge join introduces predication. -----*/
5070 [ # # # # : 0 : post_cond.add_or_replace_condition(m::Predicated(true));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5071 : : }
5072 : :
5073 : : /*----- Sort merge join does not introduce SIMD. -----*/
5074 [ # # # # : 0 : post_cond.add_condition(NoSIMD());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5075 : :
5076 : 0 : Sortedness::order_t orders;
5077 : : if constexpr (not SortLeft) {
5078 [ # # # # : 0 : Sortedness sorting_left(post_cond_children[0].get().get_condition<Sortedness>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5079 [ # # # # : 0 : orders.merge(sorting_left.orders()); // preserve sortedness of left child (including order)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5080 : 0 : }
5081 : : if constexpr (not SortRight) {
5082 [ # # # # : 0 : Sortedness sorting_right(post_cond_children[1].get().get_condition<Sortedness>());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5083 [ # # # # : 0 : orders.merge(sorting_right.orders()); // preserve sortedness of right child (including order)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5084 : 0 : }
5085 : : if constexpr (SortLeft or SortRight) {
5086 : : /*----- Decompose each clause of the join predicate of the form `A.x = B.y` into parts `A.x` and `B.y`. -----*/
5087 [ # # # # : 0 : auto [keys_parent, keys_child] = decompose_equi_predicate(M.join.predicate(), M.parent.schema());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5088 : :
5089 : : /*----- Sort merge join does sort the data on the respective key. -----*/
5090 : : if constexpr (SortLeft) {
5091 [ # # # # : 0 : for (auto &key_parent : keys_parent) {
# # # # #
# # # # #
# # ]
5092 [ # # # # : 0 : if (orders.find(key_parent) == orders.cend())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5093 [ # # # # : 0 : orders.add(key_parent, Sortedness::O_ASC); // add sortedness for left child
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5094 : : }
5095 : : }
5096 : : if constexpr (SortRight) {
5097 [ # # # # : 0 : for (auto &key_child : keys_child) {
# # # # #
# # # # #
# # ]
5098 [ # # # # : 0 : if (orders.find(key_child) == orders.cend())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5099 [ # # # # : 0 : orders.add(key_child, Sortedness::O_ASC); // add sortedness for right child
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5100 : : }
5101 : : }
5102 : 0 : }
5103 [ # # # # : 0 : post_cond.add_condition(Sortedness(std::move(orders)));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5104 : :
5105 : 0 : return post_cond;
5106 [ # # # # : 0 : }
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5107 : :
5108 : : template<bool SortLeft, bool SortRight, bool Predicated, bool CmpPredicated>
5109 : 0 : double SortMergeJoin<SortLeft, SortRight, Predicated, CmpPredicated>::cost(const Match<SortMergeJoin> &M)
5110 : : {
5111 : 0 : const double card_left = M.parent.info().estimated_cardinality;
5112 : 0 : const double card_right = M.child.info().estimated_cardinality;
5113 : :
5114 : 0 : double cost = card_left + card_right; // cost for merge
5115 : : if constexpr (SortLeft)
5116 : 0 : cost += std::log2(card_left) * card_left; // cost for sort left
5117 : : if constexpr (SortRight)
5118 : 0 : cost += std::log2(card_right) * card_right; // cost for sort right
5119 : :
5120 : 0 : return cost;
5121 : : }
5122 : :
5123 : : template<bool SortLeft, bool SortRight, bool Predicated, bool CmpPredicated>
5124 : 0 : void SortMergeJoin<SortLeft, SortRight, Predicated, CmpPredicated>::execute(
5125 : : const Match<SortMergeJoin> &M,
5126 : : setup_t setup,
5127 : : pipeline_t pipeline,
5128 : : teardown_t teardown)
5129 : : {
5130 : 0 : auto &env = CodeGenContext::Get().env();
5131 [ # # # # : 0 : const bool needs_buffer_parent = not is<const ScanOperator>(M.parent) or SortLeft;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5132 [ # # # # : 0 : const bool needs_buffer_child = not is<const ScanOperator>(M.child) or SortRight;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5133 : :
5134 : : /*----- Create infinite buffers to materialize the current results (if necessary). -----*/
5135 : 0 : M_insist(bool(M.left_materializing_factory),
5136 : : "`wasm::SortMergeJoin` must have a factory for the materialized left child");
5137 : 0 : M_insist(bool(M.right_materializing_factory),
5138 : : "`wasm::SortMergeJoin` must have a factory for the materialized right child");
5139 [ # # # # : 0 : const auto schema_parent = M.parent.schema().drop_constants().deduplicate();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5140 [ # # # # : 0 : const auto schema_child = M.child.schema().drop_constants().deduplicate();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5141 : 0 : std::optional<GlobalBuffer> buffer_parent, buffer_child;
5142 [ # # # # : 0 : if (needs_buffer_parent)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5143 [ # # # # : 0 : buffer_parent.emplace(schema_parent, *M.left_materializing_factory);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5144 [ # # # # : 0 : if (needs_buffer_child)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5145 [ # # # # : 0 : buffer_child.emplace(schema_child, *M.right_materializing_factory);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5146 : :
5147 : : /*----- Create child functions. -----*/
5148 [ # # # # : 0 : if (needs_buffer_parent) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5149 [ # # # # : 0 : FUNCTION(sort_merge_join_parent_pipeline, void(void)) // create function for parent pipeline
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5150 : : {
5151 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5152 [ # # # # : 0 : M.children[0]->execute(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5153 [ # # # # : 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ buffer_parent->setup(); }),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5154 : 0 : /* pipeline= */ [&](){ buffer_parent->consume(); },
5155 [ # # # # : 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ buffer_parent->teardown(); })
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5156 : : );
5157 : 0 : }
5158 [ # # # # : 0 : sort_merge_join_parent_pipeline(); // call parent function
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5159 : 0 : }
5160 [ # # # # : 0 : if (needs_buffer_child) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5161 [ # # # # : 0 : FUNCTION(sort_merge_join_child_pipeline, void(void)) // create function for child pipeline
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5162 : : {
5163 [ # # # # : 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5164 [ # # # # : 0 : M.children[1]->execute(
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5165 [ # # # # : 0 : /* setup= */ setup_t::Make_Without_Parent([&](){ buffer_child->setup(); }),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5166 : 0 : /* pipeline= */ [&](){ buffer_child->consume(); },
5167 [ # # # # : 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ buffer_child->teardown(); })
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5168 : : );
5169 : 0 : }
5170 [ # # # # : 0 : sort_merge_join_child_pipeline(); // call child function
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5171 : 0 : }
5172 : :
5173 : : /*----- Decompose each clause of the join predicate of the form `A.x = B.y` into parts `A.x` and `B.y`. -----*/
5174 : 0 : std::vector<SortingOperator::order_type> order_parent, order_child;
5175 [ # # # # : 0 : for (auto &clause : M.join.predicate()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5176 [ # # # # : 0 : M_insist(clause.size() == 1, "invalid equi-predicate");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5177 : 0 : auto &literal = clause[0];
5178 [ # # # # : 0 : auto &binary = as<const BinaryExpr>(literal.expr());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5179 [ # # # # : 0 : M_insist((not literal.negative() and binary.tok == TK_EQUAL) or
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5180 : : (literal.negative() and binary.tok == TK_BANG_EQUAL), "invalid equi-predicate");
5181 [ # # # # : 0 : M_insist(is<const Designator>(binary.lhs), "invalid equi-predicate");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5182 [ # # # # : 0 : M_insist(is<const Designator>(binary.rhs), "invalid equi-predicate");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5183 [ # # # # : 0 : auto [expr_parent, expr_child] = M.parent.schema().has(Schema::Identifier(*binary.lhs)) ?
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5184 [ # # # # : 0 : std::make_pair(binary.lhs.get(), binary.rhs.get()) : std::make_pair(binary.rhs.get(), binary.lhs.get());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5185 [ # # # # : 0 : order_parent.emplace_back(*expr_parent, true); // ascending order
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5186 [ # # # # : 0 : order_child.emplace_back(*expr_child, true); // ascending order
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5187 : : }
5188 [ # # # # : 0 : M_insist(order_parent.size() == order_child.size(), "number of found IDs differ");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5189 [ # # # # : 0 : M_insist(not order_parent.empty(), "must find at least one ID");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5190 : :
5191 : : /*----- If necessary, invoke sorting algorithm with buffer to sort. -----*/
5192 : : if constexpr (SortLeft)
5193 [ # # # # : 0 : quicksort<CmpPredicated>(*buffer_parent, order_parent);
# # # # #
# # # # #
# # ]
5194 : : if constexpr (SortRight)
5195 [ # # # # : 0 : quicksort<CmpPredicated>(*buffer_child, order_child);
# # # # #
# # # # #
# # ]
5196 : :
5197 : : /*----- Create predicate to check if child co-group is smaller or equal than the one of the parent relation. -----*/
5198 : 0 : auto child_smaller_equal = [&]() -> Boolx1 {
5199 : 0 : std::optional<Boolx1> child_smaller_equal_;
5200 [ # # # # : 0 : for (std::size_t i = 0; i < order_child.size(); ++i) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5201 [ # # # # : 0 : auto &des_parent = as<const Designator>(order_parent[i].first);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5202 [ # # # # : 0 : auto &des_child = as<const Designator>(order_child[i].first);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5203 [ # # # # : 0 : Token leq = Token::CreateArtificial(TK_LESS_EQUAL);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5204 [ # # # # : 0 : auto cpy_parent = std::make_unique<Designator>(des_parent.tok, des_parent.table_name, des_parent.attr_name,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5205 [ # # # # : 0 : des_parent.type(), des_parent.target());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5206 [ # # # # : 0 : auto cpy_child = std::make_unique<Designator>(des_child.tok, des_child.table_name, des_child.attr_name,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5207 [ # # # # : 0 : des_child.type(), des_child.target());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5208 [ # # # # : 0 : BinaryExpr expr(std::move(leq), std::move(cpy_child), std::move(cpy_parent));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5209 : :
5210 [ # # # # : 0 : auto child = env.get(Schema::Identifier(des_child));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5211 [ # # # # : 0 : Boolx1 cmp = env.compile<_Boolx1>(expr).is_true_and_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5212 [ # # # # : 0 : if (child_smaller_equal_)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5213 [ # # # # : 0 : child_smaller_equal_.emplace(*child_smaller_equal_ and (is_null(child) or cmp));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5214 : : else
5215 [ # # # # : 0 : child_smaller_equal_.emplace(is_null(child) or cmp);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5216 : 0 : }
5217 [ # # # # : 0 : M_insist(bool(child_smaller_equal_));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5218 [ # # # # : 0 : return *child_smaller_equal_;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5219 : 0 : };
5220 : :
5221 : : /*----- Compile data layouts to generate sequential loads from buffers. -----*/
5222 [ # # # # : 0 : static Schema empty_schema;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5223 [ # # # # : 0 : Var<U32x1> tuple_id_parent, tuple_id_child; // default initialized to 0
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5224 [ # # # # : 0 : auto [inits_parent, loads_parent, _jumps_parent] = [&](){
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5225 [ # # # # : 0 : if (needs_buffer_parent) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5226 [ # # # # : 0 : return compile_load_sequential(buffer_parent->schema(), empty_schema, buffer_parent->base_address(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5227 [ # # # # : 0 : buffer_parent->layout(), 1, buffer_parent->schema(), tuple_id_parent);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5228 : : } else {
5229 : 0 : auto &scan = as<const ScanOperator>(M.parent);
5230 [ # # # # : 0 : return compile_load_sequential(schema_parent, empty_schema, get_base_address(scan.store().table().name()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5231 [ # # # # : 0 : scan.store().table().layout(), 1, scan.store().table().schema(scan.alias()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5232 : 0 : tuple_id_parent);
5233 : : }
5234 : 0 : }();
5235 [ # # # # : 0 : auto [inits_child, loads_child, _jumps_child] = [&](){
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5236 [ # # # # : 0 : if (needs_buffer_child) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5237 [ # # # # : 0 : return compile_load_sequential(buffer_child->schema(), empty_schema, buffer_child->base_address(),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5238 [ # # # # : 0 : buffer_child->layout(), 1, buffer_child->schema(), tuple_id_child);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5239 : : } else {
5240 : 0 : auto &scan = as<const ScanOperator>(M.child);
5241 [ # # # # : 0 : return compile_load_sequential(schema_child, empty_schema, get_base_address(scan.store().table().name()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5242 [ # # # # : 0 : scan.store().table().layout(), 1, scan.store().table().schema(scan.alias()),
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5243 : 0 : tuple_id_child);
5244 : : }
5245 : 0 : }();
5246 : : /* since structured bindings cannot be used in lambda capture */
5247 [ # # # # : 0 : Block jumps_parent(std::move(_jumps_parent)), jumps_child(std::move(_jumps_child));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5248 : :
5249 : : /*----- Process both buffers together. -----*/
5250 [ # # # # : 0 : setup();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5251 [ # # # # : 0 : inits_parent.attach_to_current();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5252 [ # # # # : 0 : inits_child.attach_to_current();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5253 [ # # # # : 0 : U32x1 size_parent = needs_buffer_parent ? buffer_parent->size()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5254 [ # # # # : 0 : : get_num_rows(as<const ScanOperator>(M.parent).store().table().name());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5255 [ # # # # : 0 : U32x1 size_child = needs_buffer_child ? buffer_child->size()
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5256 [ # # # # : 0 : : get_num_rows(as<const ScanOperator>(M.child).store().table().name());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5257 [ # # # # : 0 : WHILE (tuple_id_parent < size_parent and tuple_id_child < size_child) { // neither end reached
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5258 [ # # # # : 0 : loads_parent.attach_to_current();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5259 [ # # # # : 0 : loads_child.attach_to_current();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5260 : : if constexpr (Predicated) {
5261 [ # # # # : 0 : env.add_predicate(M.join.predicate());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5262 [ # # # # : 0 : pipeline();
# # # # #
# # # # #
# # ]
5263 : : } else {
5264 [ # # # # : 0 : M_insist(CodeGenContext::Get().num_simd_lanes() == 1, "invalid number of SIMD lanes");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5265 [ # # # # : 0 : IF (env.compile<_Boolx1>(M.join.predicate()).is_true_and_not_null()) { // predicate fulfilled
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5266 : 0 : pipeline();
5267 : 0 : };
5268 : : }
5269 [ # # # # : 0 : IF (child_smaller_equal()) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
5270 : 0 : jumps_child.attach_to_current();
5271 : 0 : } ELSE {
5272 : 0 : jumps_parent.attach_to_current();
5273 : 0 : };
5274 : : }
5275 [ # # # # : 0 : teardown();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5276 : 0 : }
5277 : :
5278 : :
5279 : : /*======================================================================================================================
5280 : : * Limit
5281 : : *====================================================================================================================*/
5282 : :
5283 : 0 : ConditionSet Limit::pre_condition(std::size_t child_idx, const std::tuple<const LimitOperator*>&)
5284 : : {
5285 : 0 : M_insist(child_idx == 0);
5286 : :
5287 : 0 : ConditionSet pre_cond;
5288 : :
5289 : : /*----- Limit does not support SIMD. -----*/
5290 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
5291 : :
5292 : 0 : return pre_cond;
5293 [ # # ]: 0 : }
5294 : :
5295 : 0 : void Limit::execute(const Match<Limit> &M, setup_t setup, pipeline_t pipeline, teardown_t teardown)
5296 : : {
5297 : 0 : std::optional<Block> teardown_block; ///< block around pipeline code to jump to teardown code when limit is reached
5298 : 0 : std::optional<BlockUser> use_teardown; ///< block user to set teardown block active
5299 : :
5300 : 0 : std::optional<Var<U32x1>> counter; ///< variable to *locally* count
5301 : : /* default initialized to 0 */
5302 [ # # ]: 0 : Global<U32x1> counter_backup; ///< *global* counter backup since the following code may be called multiple times
5303 : :
5304 [ # # ]: 0 : M.child->execute(
5305 [ # # # # ]: 0 : /* setup= */ setup_t(std::move(setup), [&](){
5306 : 0 : counter.emplace(counter_backup);
5307 : 0 : teardown_block.emplace("limit.teardown", true); // create block
5308 : 0 : use_teardown.emplace(*teardown_block); // set block active s.t. it contains all following pipeline code
5309 : 0 : }),
5310 [ # # ]: 0 : /* pipeline= */ [&, pipeline=std::move(pipeline)](){
5311 : 0 : M_insist(bool(teardown_block));
5312 : 0 : M_insist(bool(counter));
5313 : 0 : const uint32_t limit = M.limit.offset() + M.limit.limit();
5314 : :
5315 : : /*----- Abort pipeline, i.e. go to teardown code, if limit is exceeded. -----*/
5316 [ # # ]: 0 : IF (*counter >= limit) {
5317 : 0 : GOTO(*teardown_block);
5318 : 0 : };
5319 : :
5320 : : /*----- Emit result if in bounds. -----*/
5321 [ # # ]: 0 : if (M.limit.offset()) {
5322 [ # # # # ]: 0 : IF (*counter >= uint32_t(M.limit.offset())) {
5323 [ # # ]: 0 : Wasm_insist(*counter < limit, "counter must not exceed limit");
5324 : 0 : pipeline();
5325 : 0 : };
5326 : 0 : } else {
5327 [ # # ]: 0 : Wasm_insist(*counter < limit, "counter must not exceed limit");
5328 : 0 : pipeline();
5329 : : }
5330 : :
5331 : : /*----- Update counter. -----*/
5332 : 0 : *counter += 1U;
5333 : 0 : },
5334 [ # # # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&, teardown=std::move(teardown)](){
5335 : 0 : M_insist(bool(teardown_block));
5336 : 0 : M_insist(bool(use_teardown));
5337 : 0 : use_teardown.reset(); // deactivate block
5338 : 0 : teardown_block.reset(); // emit block containing pipeline code into parent -> GOTO jumps here
5339 : 0 : teardown(); // *before* own teardown code to *not* jump over it in case of another limit operator
5340 : 0 : M_insist(bool(counter));
5341 : 0 : counter_backup = *counter;
5342 : 0 : counter.reset();
5343 : 0 : })
5344 : : );
5345 : 0 : }
5346 : :
5347 : :
5348 : : /*======================================================================================================================
5349 : : * Grouping combined with Join
5350 : : *====================================================================================================================*/
5351 : :
5352 : 0 : ConditionSet HashBasedGroupJoin::pre_condition(
5353 : : std::size_t child_idx,
5354 : : const std::tuple<const GroupingOperator*, const JoinOperator*, const Wildcard*, const Wildcard*>
5355 : : &partial_inner_nodes)
5356 : : {
5357 : 0 : ConditionSet pre_cond;
5358 : :
5359 : : /*----- Hash-based group-join can only be used if aggregates only depend on either build or probe relation. -----*/
5360 : 0 : auto &grouping = *std::get<0>(partial_inner_nodes);
5361 [ # # # # ]: 0 : for (auto &fn_expr : grouping.aggregates()) {
5362 [ # # ]: 0 : M_insist(fn_expr.get().args.size() <= 1);
5363 [ # # # # : 0 : if (fn_expr.get().args.size() == 1 and not is<const Designator>(fn_expr.get().args[0])) // XXX: expression with only designators from either child also valid
# # ]
5364 [ # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
5365 : : }
5366 : :
5367 : : /*----- Hash-based group-join can only be used for binary joins on equi-predicates. -----*/
5368 : 0 : auto &join = *std::get<1>(partial_inner_nodes);
5369 [ # # # # : 0 : if (not join.predicate().is_equi())
# # ]
5370 [ # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
5371 : :
5372 [ # # ]: 0 : M_insist(child_idx < 2);
5373 [ # # ]: 0 : if (child_idx == 0) {
5374 : : /*----- Decompose each clause of the join predicate of the form `A.x = B.y` into parts `A.x` and `B.y`. -----*/
5375 : 0 : auto &build = *std::get<2>(partial_inner_nodes);
5376 [ # # # # : 0 : const auto build_keys = decompose_equi_predicate(join.predicate(), build.schema()).first;
# # ]
5377 : :
5378 : : /*----- Hash-based group-join can only be used if grouping and join (i.e. build) key match (ignoring order). -*/
5379 [ # # ]: 0 : const auto num_grouping_keys = grouping.group_by().size();
5380 [ # # ]: 0 : if (num_grouping_keys != build_keys.size()) // XXX: duplicated IDs are still a match but rejected here
5381 [ # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
5382 [ # # ]: 0 : for (std::size_t i = 0; i < num_grouping_keys; ++i) {
5383 [ # # # # ]: 0 : Schema::Identifier grouping_key(grouping.group_by()[i].first.get());
5384 [ # # # # ]: 0 : if (not contains(build_keys, grouping_key))
5385 [ # # ]: 0 : return ConditionSet::Make_Unsatisfiable();
5386 [ # # ]: 0 : }
5387 [ # # ]: 0 : }
5388 : :
5389 : : /*----- Hash-based group-join does not support SIMD. -----*/
5390 [ # # # # ]: 0 : pre_cond.add_condition(NoSIMD());
5391 : :
5392 : 0 : return pre_cond;
5393 : 0 : }
5394 : :
5395 : 0 : double HashBasedGroupJoin::cost(const Match<HashBasedGroupJoin> &M)
5396 : : {
5397 : 0 : return 1.5 * M.build.info().estimated_cardinality + 1.0 * M.probe.info().estimated_cardinality +
5398 : 0 : 1.0 * M.join.info().estimated_cardinality;
5399 : : }
5400 : :
5401 : 0 : ConditionSet HashBasedGroupJoin::post_condition(const Match<HashBasedGroupJoin>&)
5402 : : {
5403 : 0 : ConditionSet post_cond;
5404 : :
5405 : : /*----- Hash-based group-join does not introduce predication (it is already handled by the hash table). -----*/
5406 [ # # # # ]: 0 : post_cond.add_condition(Predicated(false));
5407 : :
5408 : : /*----- Hash-based group-join does not introduce SIMD. -----*/
5409 [ # # # # ]: 0 : post_cond.add_condition(NoSIMD());
5410 : :
5411 : 0 : return post_cond;
5412 [ # # ]: 0 : }
5413 : :
5414 : 0 : void HashBasedGroupJoin::execute(const Match<HashBasedGroupJoin> &M, setup_t setup, pipeline_t pipeline,
5415 : : teardown_t teardown)
5416 : : {
5417 : : // TODO: determine setup
5418 : 0 : const uint64_t AGGREGATES_SIZE_THRESHOLD_IN_BITS =
5419 : 0 : M.use_in_place_values ? std::numeric_limits<uint64_t>::max() : 0;
5420 : :
5421 : 0 : auto &C = Catalog::Get();
5422 : 0 : const auto num_keys = M.grouping.group_by().size();
5423 : :
5424 : : /*----- Compute hash table schema and information about aggregates, especially AVG aggregates. -----*/
5425 : 0 : Schema ht_schema;
5426 [ # # ]: 0 : for (std::size_t i = 0; i < num_keys; ++i) {
5427 [ # # # # ]: 0 : auto &e = M.grouping.schema()[i];
5428 [ # # # # ]: 0 : ht_schema.add(e.id, e.type, e.constraints);
5429 : 0 : }
5430 [ # # # # : 0 : auto aggregates_info = compute_aggregate_info(M.grouping.aggregates(), M.grouping.schema(), num_keys);
# # ]
5431 : 0 : const auto &aggregates = aggregates_info.first;
5432 : 0 : const auto &avg_aggregates = aggregates_info.second;
5433 : 0 : bool needs_build_counter = false; ///< flag whether additional COUNT per group during build phase must be emitted
5434 : 0 : uint64_t aggregates_size_in_bits = 0;
5435 [ # # ]: 0 : for (auto &info : aggregates) {
5436 [ # # # # ]: 0 : ht_schema.add(info.entry);
5437 [ # # ]: 0 : aggregates_size_in_bits += info.entry.type->size();
5438 : :
5439 : : /* Add additional COUNT per group during build phase if COUNT or SUM dependent on probe relation occurs. */
5440 [ # # # # ]: 0 : if (info.fnid == m::Function::FN_COUNT or info.fnid == m::Function::FN_SUM) {
5441 [ # # ]: 0 : if (not info.args.empty()) {
5442 [ # # ]: 0 : M_insist(info.args.size() == 1, "aggregate functions expect at most one argument");
5443 [ # # ]: 0 : auto &des = as<const Designator>(*info.args[0]);
5444 [ # # # # : 0 : Schema::Identifier arg(des.table_name.text, des.attr_name.text.assert_not_none());
# # ]
5445 [ # # # # : 0 : if (M.probe.schema().has(arg))
# # ]
5446 : 0 : needs_build_counter = true;
5447 : 0 : }
5448 : 0 : }
5449 : : }
5450 [ # # ]: 0 : if (needs_build_counter) {
5451 [ # # # # : 0 : ht_schema.add(Schema::Identifier(C.pool("$build_counter")), Type::Get_Integer(Type::TY_Scalar, 8),
# # # # #
# ]
5452 : : Schema::entry_type::NOT_NULLABLE);
5453 : 0 : aggregates_size_in_bits += 64;
5454 : 0 : }
5455 [ # # # # : 0 : ht_schema.add(Schema::Identifier(C.pool("$probe_counter")), Type::Get_Integer(Type::TY_Scalar, 8),
# # # # #
# ]
5456 : : Schema::entry_type::NOT_NULLABLE);
5457 : 0 : aggregates_size_in_bits += 64;
5458 : :
5459 : : /*----- Decompose each clause of the join predicate of the form `A.x = B.y` into parts `A.x` and `B.y`. -----*/
5460 [ # # # # : 0 : const auto [build_keys, probe_keys] = decompose_equi_predicate(M.join.predicate(), M.build.schema());
# # ]
5461 [ # # ]: 0 : M_insist(build_keys.size() == num_keys);
5462 : :
5463 : : /*----- Compute initial capacity of hash table. -----*/
5464 [ # # ]: 0 : uint32_t initial_capacity = compute_initial_ht_capacity(M.grouping, M.load_factor);
5465 : :
5466 : : /*----- Create hash table for build relation. -----*/
5467 : 0 : std::unique_ptr<HashTable> ht;
5468 [ # # ]: 0 : std::vector<HashTable::index_t> key_indices(num_keys);
5469 [ # # ]: 0 : std::iota(key_indices.begin(), key_indices.end(), 0);
5470 [ # # ]: 0 : if (M.use_open_addressing_hashing) {
5471 [ # # ]: 0 : if (aggregates_size_in_bits < AGGREGATES_SIZE_THRESHOLD_IN_BITS)
5472 [ # # ]: 0 : ht = std::make_unique<GlobalOpenAddressingInPlaceHashTable>(ht_schema, std::move(key_indices),
5473 : : initial_capacity);
5474 : : else
5475 [ # # ]: 0 : ht = std::make_unique<GlobalOpenAddressingOutOfPlaceHashTable>(ht_schema, std::move(key_indices),
5476 : : initial_capacity);
5477 [ # # ]: 0 : if (M.use_quadratic_probing)
5478 [ # # # # ]: 0 : as<OpenAddressingHashTableBase>(*ht).set_probing_strategy<QuadraticProbing>();
5479 : : else
5480 [ # # # # ]: 0 : as<OpenAddressingHashTableBase>(*ht).set_probing_strategy<LinearProbing>();
5481 : 0 : } else {
5482 [ # # ]: 0 : ht = std::make_unique<GlobalChainedHashTable>(ht_schema, std::move(key_indices), initial_capacity);
5483 : : }
5484 : :
5485 : 0 : std::optional<HashTable::entry_t> dummy; ///< *local* dummy slot
5486 : :
5487 : : /** Helper function to compute aggregates to be stored in \p entry given the arguments contained in environment \p
5488 : : * env for the phase (i.e. build or probe) with the schema \p schema. The flag \p build_phase determines which
5489 : : * phase is currently active.
5490 : : *
5491 : : * Returns three code blocks: the first one initializes all aggregates, the second one updates all but the AVG
5492 : : * aggregates, and the third one updates the AVG aggregates. */
5493 : 0 : auto compile_aggregates = [&](HashTable::entry_t &entry, const Environment &env, const Schema &schema,
5494 : : bool build_phase) -> std::tuple<Block, Block, Block>
5495 : : {
5496 : 0 : Block init_aggs("hash_based_group_join.init_aggs", false),
5497 [ # # ]: 0 : update_aggs("hash_based_group_join.update_aggs", false),
5498 [ # # ]: 0 : update_avg_aggs("hash_based_group_join.update_avg_aggs", false);
5499 [ # # ]: 0 : for (auto &info : aggregates) {
5500 : 0 : bool is_min = false; ///< flag to indicate whether aggregate function is MIN
5501 [ # # # # : 0 : switch (info.fnid) {
# # ]
5502 : : default:
5503 [ # # ]: 0 : M_unreachable("unsupported aggregate function");
5504 : : case m::Function::FN_MIN:
5505 : 0 : is_min = true; // set flag and delegate to MAX case
5506 : : case m::Function::FN_MAX: {
5507 [ # # ]: 0 : M_insist(info.args.size() == 1, "MIN and MAX aggregate functions expect exactly one argument");
5508 [ # # ]: 0 : auto &arg = as<const Designator>(*info.args[0]);
5509 [ # # # # : 0 : const bool bound = schema.has(Schema::Identifier(arg.table_name.text,
# # ]
5510 [ # # ]: 0 : arg.attr_name.text.assert_not_none()));
5511 : :
5512 [ # # ]: 0 : std::visit(overloaded {
5513 : 0 : [&]<sql_type _T>(HashTable::reference_t<_T> &&r) -> void
5514 : : requires (not (std::same_as<_T, _Boolx1> or std::same_as<_T, NChar>)) {
5515 : : using type = typename _T::type;
5516 : : using T = PrimitiveExpr<type>;
5517 : :
5518 [ # # # # : 0 : if (build_phase) {
# # # # #
# # # ]
5519 : 0 : BLOCK_OPEN(init_aggs) {
5520 [ # # # # : 0 : auto neutral = is_min ? T(std::numeric_limits<type>::max())
# # # # #
# # # # #
# # # # #
# # # #
# ]
5521 [ # # # # : 0 : : T(std::numeric_limits<type>::lowest());
# # # # #
# # # ]
5522 [ # # # # : 0 : if (bound) {
# # # # #
# # # ]
5523 [ # # # # : 0 : auto _arg = env.compile(arg);
# # # # #
# # # ]
5524 [ # # # # : 0 : auto [val_, is_null] = convert<_T>(_arg).split();
# # # # #
# # # # #
# # # # #
# # # #
# ]
5525 [ # # # # : 0 : T val(val_); // due to structured binding and lambda closure
# # # # #
# # # ]
5526 [ # # # # : 0 : IF (is_null) {
# # # # #
# # # # #
# # # # #
# # # #
# ]
5527 [ # # # # : 0 : r.clone().set_value(neutral); // initialize with neutral element +inf or -inf
# # # # #
# # # # #
# # # # #
# # # #
# ]
5528 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
5529 [ # # # # : 0 : r.clone().set_null_bit(Boolx1(true)); // first value is NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
5530 [ # # # # : 0 : } ELSE {
# # # # #
# # # ]
5531 [ # # # # : 0 : r.clone().set_value(val); // initialize with first value
# # # # #
# # # # #
# # # # #
# # # #
# ]
5532 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
5533 [ # # # # : 0 : r.clone().set_null_bit(Boolx1(false)); // first value is not NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
5534 : 0 : };
5535 : 0 : } else {
5536 [ # # # # : 0 : r.clone().set_value(neutral); // initialize with neutral element +inf or -inf
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5537 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
5538 [ # # # # : 0 : r.clone().set_null_bit(Boolx1(true)); // initialize with neutral element NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5539 : : }
5540 : 0 : }
5541 : 0 : }
5542 [ # # # # : 0 : if (not bound) {
# # # # #
# # # ]
5543 : 0 : r.discard();
5544 : 0 : return; // MIN and MAX does not change in phase when argument is unbound
5545 : : }
5546 : 0 : BLOCK_OPEN(update_aggs) {
5547 [ # # # # : 0 : auto _arg = env.compile(arg);
# # # # #
# # # ]
5548 [ # # # # : 0 : _T _new_val = convert<_T>(_arg);
# # # # #
# # # ]
5549 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # ]
5550 [ # # # # : 0 : auto [new_val_, new_val_is_null_] = _new_val.split();
# # # # #
# # # ]
5551 [ # # # # : 0 : auto [old_min_max_, old_min_max_is_null] = _T(r.clone()).split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5552 [ # # # # : 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # ]
5553 : :
5554 [ # # # # : 0 : auto chosen_r = Select(new_val_is_null, dummy->extract<_T>(info.entry.id), r.clone());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5555 : : if constexpr (std::floating_point<type>) {
5556 [ # # # # : 0 : chosen_r.set_value(
# # # # #
# # # # #
# # # # #
# ]
5557 [ # # # # : 0 : is_min ? min(old_min_max_, new_val_) // update old min with new value
# # # # ]
5558 [ # # # # ]: 0 : : max(old_min_max_, new_val_) // update old max with new value
5559 : : ); // if new value is NULL, only dummy is written
5560 : : } else {
5561 [ # # # # : 0 : const Var<T> new_val(new_val_),
# # # # ]
5562 [ # # # # : 0 : old_min_max(old_min_max_); // due to multiple uses
# # # # ]
5563 [ # # # # : 0 : auto cmp = is_min ? new_val < old_min_max : new_val > old_min_max;
# # # # #
# # # # #
# # # # #
# # # #
# ]
5564 [ # # # # : 0 : chosen_r.set_value(
# # # # ]
5565 [ # # # # : 0 : Select(cmp,
# # # # ]
5566 : : new_val, // update to new value
5567 : : old_min_max) // do not update
5568 : : ); // if new value is NULL, only dummy is written
5569 : 0 : }
5570 [ # # # # : 0 : r.set_null_bit(
# # # # #
# # # ]
5571 [ # # # # : 0 : old_min_max_is_null and new_val_is_null // MIN/MAX is NULL iff all values are NULL
# # # # #
# # # ]
5572 : : );
5573 : 0 : } else {
5574 [ # # # # : 0 : auto new_val_ = _new_val.insist_not_null();
# # # # #
# # # ]
5575 [ # # # # : 0 : auto old_min_max_ = _T(r.clone()).insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5576 : : if constexpr (std::floating_point<type>) {
5577 [ # # # # : 0 : r.set_value(
# # # # #
# # # # #
# # # # #
# ]
5578 [ # # # # : 0 : is_min ? min(old_min_max_, new_val_) // update old min with new value
# # # # ]
5579 [ # # # # ]: 0 : : max(old_min_max_, new_val_) // update old max with new value
5580 : : );
5581 : : } else {
5582 [ # # # # : 0 : const Var<T> new_val(new_val_),
# # # # ]
5583 [ # # # # : 0 : old_min_max(old_min_max_); // due to multiple uses
# # # # ]
5584 [ # # # # : 0 : auto cmp = is_min ? new_val < old_min_max : new_val > old_min_max;
# # # # #
# # # # #
# # # # #
# # # #
# ]
5585 [ # # # # : 0 : r.set_value(
# # # # ]
5586 [ # # # # : 0 : Select(cmp,
# # # # ]
5587 : : new_val, // update to new value
5588 : : old_min_max) // do not update
5589 : : );
5590 : 0 : }
5591 : : /* do not update NULL bit since it is already set to `false` */
5592 : 0 : }
5593 : 0 : }
5594 : 0 : },
5595 : 0 : []<sql_type _T>(HashTable::reference_t<_T>&&) -> void
5596 : : requires std::same_as<_T,_Boolx1> or std::same_as<_T, NChar> {
5597 : 0 : M_unreachable("invalid type");
5598 : : },
5599 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
5600 [ # # ]: 0 : }, entry.extract(info.entry.id));
5601 : 0 : break;
5602 : : }
5603 : : case m::Function::FN_AVG: {
5604 [ # # ]: 0 : auto it = avg_aggregates.find(info.entry.id);
5605 [ # # ]: 0 : M_insist(it != avg_aggregates.end());
5606 : 0 : const auto &avg_info = it->second;
5607 [ # # ]: 0 : M_insist(avg_info.compute_running_avg,
5608 : : "AVG aggregate may only occur for running average computations");
5609 [ # # ]: 0 : M_insist(info.args.size() == 1, "AVG aggregate function expects exactly one argument");
5610 [ # # ]: 0 : auto &arg = as<const Designator>(*info.args[0]);
5611 [ # # # # : 0 : const bool bound = schema.has(Schema::Identifier(arg.table_name.text,
# # ]
5612 [ # # ]: 0 : arg.attr_name.text.assert_not_none()));
5613 : :
5614 [ # # ]: 0 : auto r = entry.extract<_Doublex1>(info.entry.id);
5615 : :
5616 [ # # ]: 0 : if (build_phase) {
5617 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
5618 [ # # ]: 0 : if (bound) {
5619 [ # # ]: 0 : auto _arg = env.compile(arg);
5620 [ # # # # ]: 0 : auto [val_, is_null] = convert<_Doublex1>(_arg).split();
5621 [ # # ]: 0 : Doublex1 val(val_); // due to structured binding and lambda closure
5622 [ # # ]: 0 : IF (is_null) {
5623 [ # # # # ]: 0 : r.clone().set_value(Doublex1(0.0)); // initialize with neutral element 0
5624 [ # # ]: 0 : if (info.entry.nullable())
5625 [ # # # # ]: 0 : r.clone().set_null_bit(Boolx1(true)); // first value is NULL
5626 [ # # ]: 0 : } ELSE {
5627 [ # # # # ]: 0 : r.clone().set_value(val); // initialize with first value
5628 [ # # ]: 0 : if (info.entry.nullable())
5629 [ # # # # ]: 0 : r.clone().set_null_bit(Boolx1(false)); // first value is not NULL
5630 : 0 : };
5631 : 0 : } else {
5632 [ # # # # : 0 : r.clone().set_value(Doublex1(0.0)); // initialize with neutral element 0
# # ]
5633 [ # # ]: 0 : if (info.entry.nullable())
5634 [ # # # # : 0 : r.clone().set_null_bit(Boolx1(true)); // initialize with neutral element NULL
# # ]
5635 : : }
5636 : : }
5637 : 0 : }
5638 [ # # ]: 0 : if (not bound) {
5639 [ # # ]: 0 : r.discard();
5640 : 0 : break; // AVG does not change in phase when argument is unbound
5641 : : }
5642 [ # # ]: 0 : BLOCK_OPEN(update_avg_aggs) {
5643 : : /* Compute AVG as iterative mean as described in Knuth, The Art of Computer Programming
5644 : : * Vol 2, section 4.2.2. */
5645 [ # # ]: 0 : auto _arg = env.compile(arg);
5646 [ # # ]: 0 : _Doublex1 _new_val = convert<_Doublex1>(_arg);
5647 [ # # ]: 0 : if (_new_val.can_be_null()) {
5648 [ # # ]: 0 : auto [new_val, new_val_is_null_] = _new_val.split();
5649 [ # # # # : 0 : auto [old_avg_, old_avg_is_null] = _Doublex1(r.clone()).split();
# # ]
5650 [ # # ]: 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
5651 [ # # ]: 0 : const Var<Doublex1> old_avg(old_avg_); // due to multiple uses
5652 : :
5653 [ # # ]: 0 : auto delta_absolute = new_val - old_avg;
5654 [ # # # # : 0 : auto running_count = _I64x1(entry.get<_I64x1>(avg_info.running_count)).insist_not_null();
# # ]
5655 [ # # # # ]: 0 : auto delta_relative = delta_absolute / running_count.to<double>();
5656 : :
5657 [ # # # # : 0 : auto chosen_r = Select(new_val_is_null, dummy->extract<_Doublex1>(info.entry.id), r.clone());
# # # # ]
5658 [ # # ]: 0 : chosen_r.set_value(
5659 [ # # ]: 0 : old_avg + delta_relative // update old average with new value
5660 : : ); // if new value is NULL, only dummy is written
5661 [ # # ]: 0 : r.set_null_bit(
5662 [ # # ]: 0 : old_avg_is_null and new_val_is_null // AVG is NULL iff all values are NULL
5663 : : );
5664 : 0 : } else {
5665 [ # # ]: 0 : auto new_val = _new_val.insist_not_null();
5666 [ # # # # : 0 : auto old_avg_ = _Doublex1(r.clone()).insist_not_null();
# # ]
5667 [ # # ]: 0 : const Var<Doublex1> old_avg(old_avg_); // due to multiple uses
5668 : :
5669 [ # # ]: 0 : auto delta_absolute = new_val - old_avg;
5670 [ # # # # : 0 : auto running_count = _I64x1(entry.get<_I64x1>(avg_info.running_count)).insist_not_null();
# # ]
5671 [ # # # # ]: 0 : auto delta_relative = delta_absolute / running_count.to<double>();
5672 [ # # ]: 0 : r.set_value(
5673 [ # # ]: 0 : old_avg + delta_relative // update old average with new value
5674 : : );
5675 : : /* do not update NULL bit since it is already set to `false` */
5676 : 0 : }
5677 : 0 : }
5678 : 0 : break;
5679 : 0 : }
5680 : : case m::Function::FN_SUM: {
5681 [ # # ]: 0 : M_insist(info.args.size() == 1, "SUM aggregate function expects exactly one argument");
5682 [ # # ]: 0 : auto &arg = as<const Designator>(*info.args[0]);
5683 [ # # # # : 0 : const bool bound = schema.has(Schema::Identifier(arg.table_name.text,
# # ]
5684 [ # # ]: 0 : arg.attr_name.text.assert_not_none()));
5685 : :
5686 [ # # ]: 0 : std::visit(overloaded {
5687 : 0 : [&]<sql_type _T>(HashTable::reference_t<_T> &&r) -> void
5688 : : requires (not (std::same_as<_T, _Boolx1> or std::same_as<_T, NChar>)) {
5689 : : using type = typename _T::type;
5690 : : using T = PrimitiveExpr<type>;
5691 : :
5692 [ # # # # : 0 : if (build_phase) {
# # # # #
# # # ]
5693 : 0 : BLOCK_OPEN(init_aggs) {
5694 [ # # # # : 0 : if (bound) {
# # # # #
# # # ]
5695 [ # # # # : 0 : auto _arg = env.compile(arg);
# # # # #
# # # ]
5696 [ # # # # : 0 : auto [val_, is_null] = convert<_T>(_arg).split();
# # # # #
# # # # #
# # # # #
# # # #
# ]
5697 [ # # # # : 0 : T val(val_); // due to structured binding and lambda closure
# # # # #
# # # ]
5698 [ # # # # : 0 : IF (is_null) {
# # # # #
# # # ]
5699 [ # # # # : 0 : r.clone().set_value(T(type(0))); // initialize with neutral element 0
# # # # #
# # # # #
# # # # #
# # # #
# ]
5700 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
5701 [ # # # # : 0 : r.clone().set_null_bit(Boolx1(true)); // first value is NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
5702 [ # # # # : 0 : } ELSE {
# # # # #
# # # ]
5703 [ # # # # : 0 : r.clone().set_value(val); // initialize with first value
# # # # #
# # # # #
# # # # #
# # # #
# ]
5704 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
5705 [ # # # # : 0 : r.clone().set_null_bit(Boolx1(false)); // first value is not NULL
# # # # #
# # # # #
# # # # #
# # # #
# ]
5706 : 0 : };
5707 : 0 : } else {
5708 [ # # # # : 0 : r.clone().set_value(T(type(0))); // initialize with neutral element 0
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5709 [ # # # # : 0 : if (info.entry.nullable())
# # # # #
# # # ]
5710 [ # # # # : 0 : r.clone().set_null_bit(Boolx1(true)); // initialize with neutral element NULL
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5711 : : }
5712 : : }
5713 : 0 : }
5714 [ # # # # : 0 : if (not bound) {
# # # # #
# # # ]
5715 : 0 : r.discard();
5716 : 0 : return; // SUM may later be multiplied with group counter but does not change here
5717 : : }
5718 : 0 : BLOCK_OPEN(update_aggs) {
5719 [ # # # # : 0 : auto _arg = env.compile(arg);
# # # # #
# # # ]
5720 [ # # # # : 0 : _T _new_val = convert<_T>(_arg);
# # # # #
# # # ]
5721 [ # # # # : 0 : if (_new_val.can_be_null()) {
# # # # #
# # # ]
5722 [ # # # # : 0 : auto [new_val, new_val_is_null_] = _new_val.split();
# # # # #
# # # ]
5723 [ # # # # : 0 : auto [old_sum, old_sum_is_null] = _T(r.clone()).split();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5724 [ # # # # : 0 : const Var<Boolx1> new_val_is_null(new_val_is_null_); // due to multiple uses
# # # # #
# # # ]
5725 : :
5726 [ # # # # : 0 : auto chosen_r = Select(new_val_is_null, dummy->extract<_T>(info.entry.id), r.clone());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5727 [ # # # # : 0 : chosen_r.set_value(
# # # # #
# # # ]
5728 [ # # # # : 0 : old_sum + new_val // add new value to old sum
# # # # #
# # # # #
# # # # #
# # # #
# ]
5729 : : ); // if new value is NULL, only dummy is written
5730 [ # # # # : 0 : r.set_null_bit(
# # # # #
# # # ]
5731 [ # # # # : 0 : old_sum_is_null and new_val_is_null // SUM is NULL iff all values are NULL
# # # # #
# # # ]
5732 : : );
5733 : 0 : } else {
5734 [ # # # # : 0 : auto new_val = _new_val.insist_not_null();
# # # # #
# # # ]
5735 [ # # # # : 0 : auto old_sum = _T(r.clone()).insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5736 [ # # # # : 0 : r.set_value(
# # # # #
# # # ]
5737 [ # # # # : 0 : old_sum + new_val // add new value to old sum
# # # # #
# # # # #
# # # # #
# # # #
# ]
5738 : : );
5739 : : /* do not update NULL bit since it is already set to `false` */
5740 : 0 : }
5741 : 0 : }
5742 : 0 : },
5743 : 0 : []<sql_type _T>(HashTable::reference_t<_T>&&) -> void
5744 : : requires std::same_as<_T,_Boolx1> or std::same_as<_T, NChar> {
5745 : 0 : M_unreachable("invalid type");
5746 : : },
5747 : 0 : [](std::monostate) -> void { M_unreachable("invalid reference"); },
5748 [ # # ]: 0 : }, entry.extract(info.entry.id));
5749 : 0 : break;
5750 : : }
5751 : : case m::Function::FN_COUNT: {
5752 [ # # ]: 0 : M_insist(info.args.size() <= 1, "COUNT aggregate function expects at most one argument");
5753 : :
5754 [ # # ]: 0 : auto r = entry.get<_I64x1>(info.entry.id); // do not extract to be able to access for AVG case
5755 : :
5756 [ # # ]: 0 : if (info.args.empty()) {
5757 [ # # ]: 0 : if (not build_phase) {
5758 [ # # ]: 0 : r.discard();
5759 : 0 : break; // COUNT(*) will later be multiplied with probe counter but only changes in build phase
5760 : : }
5761 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
5762 [ # # # # : 0 : r.clone() = _I64x1(1); // initialize with 1 (for first value)
# # ]
5763 : : }
5764 [ # # ]: 0 : BLOCK_OPEN(update_aggs) {
5765 [ # # # # : 0 : auto old_count = _I64x1(r.clone()).insist_not_null();
# # ]
5766 [ # # ]: 0 : r.set_value(
5767 [ # # ]: 0 : old_count + int64_t(1) // increment old count by 1
5768 : : );
5769 : : /* do not update NULL bit since it is already set to `false` */
5770 : 0 : }
5771 : 0 : } else {
5772 [ # # ]: 0 : auto &arg = as<const Designator>(*info.args[0]);
5773 [ # # # # : 0 : const bool bound = schema.has(Schema::Identifier(arg.table_name.text,
# # ]
5774 [ # # ]: 0 : arg.attr_name.text.assert_not_none()));
5775 : :
5776 [ # # ]: 0 : if (build_phase) {
5777 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
5778 [ # # ]: 0 : if (bound) {
5779 [ # # ]: 0 : auto _arg = env.compile(arg);
5780 : : I64x1 new_val_not_null =
5781 [ # # # # : 0 : can_be_null(_arg) ? not_null(_arg).to<int64_t>()
# # # # #
# # # ]
5782 [ # # # # ]: 0 : : (discard(_arg), I64x1(1)); // discard since no use
5783 [ # # # # : 0 : r.clone() = _I64x1(new_val_not_null); // initialize with 1 iff first value is present
# # # # ]
5784 : 0 : } else {
5785 [ # # # # : 0 : r.clone() = _I64x1(0); // initialize with neutral element 0
# # ]
5786 : : }
5787 : : }
5788 : 0 : }
5789 [ # # ]: 0 : if (not bound) {
5790 [ # # ]: 0 : r.discard();
5791 : 0 : break; // COUNT may later be multiplied with group counter but does not change here
5792 : : }
5793 [ # # ]: 0 : BLOCK_OPEN(update_aggs) {
5794 [ # # ]: 0 : auto _arg = env.compile(arg);
5795 : : I64x1 new_val_not_null =
5796 [ # # # # : 0 : can_be_null(_arg) ? not_null(_arg).to<int64_t>()
# # # # #
# # # ]
5797 [ # # # # ]: 0 : : (discard(_arg), I64x1(1)); // discard since no use
5798 [ # # # # : 0 : auto old_count = _I64x1(r.clone()).insist_not_null();
# # ]
5799 [ # # ]: 0 : r.set_value(
5800 [ # # # # ]: 0 : old_count + new_val_not_null // increment old count by 1 iff new value is present
5801 : : );
5802 : : /* do not update NULL bit since it is already set to `false` */
5803 : 0 : }
5804 : : }
5805 : 0 : break;
5806 : 0 : }
5807 : : }
5808 : : }
5809 [ # # ]: 0 : return { std::move(init_aggs), std::move(update_aggs), std::move(update_avg_aggs) };
5810 : 0 : };
5811 : :
5812 : : /*----- Create function for build child. -----*/
5813 [ # # # # : 0 : FUNCTION(hash_based_group_join_build_child_pipeline, void(void)) // create function for pipeline
# # # # ]
5814 : : {
5815 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
5816 : :
5817 [ # # ]: 0 : M.children[0]->execute(
5818 [ # # # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){
5819 : 0 : ht->setup();
5820 : 0 : ht->set_high_watermark(M.load_factor);
5821 : 0 : dummy.emplace(ht->dummy_entry()); // create dummy slot to ignore NULL values in aggregate computations
5822 : 0 : }),
5823 [ # # # # ]: 0 : /* pipeline= */ [&](){
5824 : 0 : M_insist(bool(dummy));
5825 : 0 : const auto &env = CodeGenContext::Get().env();
5826 : :
5827 : 0 : std::optional<Boolx1> build_key_not_null;
5828 [ # # ]: 0 : for (auto &build_key : build_keys) {
5829 [ # # ]: 0 : auto val = env.get(build_key);
5830 [ # # ]: 0 : if (build_key_not_null)
5831 [ # # # # : 0 : build_key_not_null.emplace(*build_key_not_null and not_null(val));
# # ]
5832 : : else
5833 [ # # # # ]: 0 : build_key_not_null.emplace(not_null(val));
5834 : 0 : }
5835 [ # # ]: 0 : M_insist(bool(build_key_not_null));
5836 [ # # # # ]: 0 : IF (*build_key_not_null) { // TODO: predicated version
5837 : : /*----- Insert key if not yet done. -----*/
5838 : 0 : std::vector<SQL_t> key;
5839 [ # # ]: 0 : for (auto &build_key : build_keys)
5840 [ # # # # ]: 0 : key.emplace_back(env.get(build_key));
5841 [ # # ]: 0 : auto [entry, inserted] = ht->try_emplace(std::move(key));
5842 : :
5843 : : /*----- Compile aggregates. -----*/
5844 [ # # # # ]: 0 : auto t = compile_aggregates(entry, env, M.build.schema(), /* build_phase= */ true);
5845 : 0 : auto &init_aggs = std::get<0>(t);
5846 : 0 : auto &update_aggs = std::get<1>(t);
5847 : 0 : auto &update_avg_aggs = std::get<2>(t);
5848 : :
5849 : : /*----- Add group counters to compiled aggregates. -----*/
5850 [ # # ]: 0 : if (needs_build_counter) {
5851 [ # # # # : 0 : auto r = entry.extract<_I64x1>(C.pool("$build_counter"));
# # ]
5852 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
5853 [ # # # # : 0 : r.clone() = _I64x1(1); // initialize with 1 (for first value)
# # ]
5854 : : }
5855 [ # # ]: 0 : BLOCK_OPEN(update_aggs) {
5856 [ # # # # : 0 : auto old_count = _I64x1(r.clone()).insist_not_null();
# # ]
5857 [ # # ]: 0 : r.set_value(
5858 [ # # ]: 0 : old_count + int64_t(1) // increment old count by 1
5859 : : );
5860 : : /* do not update NULL bit since it is already set to `false` */
5861 : 0 : }
5862 : 0 : }
5863 [ # # ]: 0 : BLOCK_OPEN(init_aggs) {
5864 [ # # # # : 0 : auto r = entry.extract<_I64x1>(C.pool("$probe_counter"));
# # ]
5865 [ # # # # ]: 0 : r = _I64x1(0); // initialize with neutral element 0
5866 : 0 : }
5867 : :
5868 : : /*----- If group has been inserted, initialize aggregates. Otherwise, update them. -----*/
5869 [ # # ]: 0 : IF (inserted) {
5870 : 0 : init_aggs.attach_to_current();
5871 : 0 : } ELSE {
5872 : 0 : update_aggs.attach_to_current();
5873 : 0 : update_avg_aggs.attach_to_current(); // after others to ensure that running count is incremented before
5874 : 0 : };
5875 : 0 : };
5876 : 0 : },
5877 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ ht->teardown(); })
5878 : : );
5879 : 0 : }
5880 [ # # ]: 0 : hash_based_group_join_build_child_pipeline(); // call build child function
5881 : :
5882 : : /*----- Create function for probe child. -----*/
5883 [ # # # # : 0 : FUNCTION(hash_based_group_join_probe_child_pipeline, void(void)) // create function for pipeline
# # # # ]
5884 : : {
5885 [ # # # # ]: 0 : auto S = CodeGenContext::Get().scoped_environment(); // create scoped environment for this function
5886 : :
5887 [ # # ]: 0 : M.children[1]->execute(
5888 [ # # ]: 0 : /* setup= */ setup_t::Make_Without_Parent([&](){
5889 : 0 : ht->setup();
5890 : 0 : dummy.emplace(ht->dummy_entry()); // create dummy slot to ignore NULL values in aggregate computations
5891 : 0 : }),
5892 [ # # # # ]: 0 : /* pipeline= */ [&](){
5893 : 0 : M_insist(bool(dummy));
5894 : 0 : const auto &env = CodeGenContext::Get().env();
5895 : :
5896 : : /* TODO: may check for NULL on probe keys as well, branching + predicated version */
5897 : : /*----- Probe with probe key. -----*/
5898 : 0 : std::vector<SQL_t> key;
5899 [ # # ]: 0 : for (auto &probe_key : probe_keys)
5900 [ # # # # ]: 0 : key.emplace_back(env.get(probe_key));
5901 [ # # ]: 0 : auto [entry, found] = ht->find(std::move(key));
5902 : :
5903 : : /*----- Compile aggregates. -----*/
5904 [ # # # # ]: 0 : auto t = compile_aggregates(entry, env, M.probe.schema(), /* build_phase= */ false);
5905 : 0 : auto &init_aggs = std::get<0>(t);
5906 : 0 : auto &update_aggs = std::get<1>(t);
5907 : 0 : auto &update_avg_aggs = std::get<2>(t);
5908 : :
5909 : : /*----- Add probe counter to compiled aggregates. -----*/
5910 [ # # ]: 0 : BLOCK_OPEN(update_aggs) {
5911 [ # # # # : 0 : auto r = entry.extract<_I64x1>(C.pool("$probe_counter"));
# # ]
5912 [ # # # # : 0 : auto old_count = _I64x1(r.clone()).insist_not_null();
# # ]
5913 [ # # ]: 0 : r.set_value(
5914 [ # # ]: 0 : old_count + int64_t(1) // increment old count by 1
5915 : : );
5916 : : /* do not update NULL bit since it is already set to `false` */
5917 : 0 : }
5918 : :
5919 : : /*----- If group has been inserted, initialize aggregates. Otherwise, update them. -----*/
5920 [ # # # # ]: 0 : M_insist(init_aggs.empty(), "aggregates must be initialized in build phase");
5921 [ # # ]: 0 : IF (found) {
5922 : 0 : update_aggs.attach_to_current();
5923 : 0 : update_avg_aggs.attach_to_current(); // after others to ensure that running count is incremented before
5924 : 0 : };
5925 : 0 : },
5926 [ # # ]: 0 : /* teardown= */ teardown_t::Make_Without_Parent([&](){ ht->teardown(); })
5927 : : );
5928 : 0 : }
5929 [ # # ]: 0 : hash_based_group_join_probe_child_pipeline(); // call probe child function
5930 : :
5931 [ # # # # ]: 0 : auto &env = CodeGenContext::Get().env();
5932 : :
5933 : : /*----- Process each computed group. -----*/
5934 [ # # # # ]: 0 : setup_t(std::move(setup), [&](){ ht->setup(); })();
5935 [ # # # # ]: 0 : ht->for_each([&, pipeline=std::move(pipeline)](HashTable::const_entry_t entry){
5936 : : /*----- Check whether probe match was found. -----*/
5937 [ # # # # : 0 : I64x1 probe_counter = _I64x1(entry.get<_I64x1>(C.pool("$probe_counter"))).insist_not_null();
# # # # ]
5938 [ # # # # : 0 : IF (probe_counter != int64_t(0)) {
# # ]
5939 : : /*----- Compute key schema to detect duplicated keys. -----*/
5940 : 0 : Schema key_schema;
5941 [ # # ]: 0 : for (std::size_t i = 0; i < num_keys; ++i) {
5942 [ # # ]: 0 : auto &e = M.grouping.schema()[i];
5943 [ # # # # ]: 0 : key_schema.add(e.id, e.type, e.constraints);
5944 : 0 : }
5945 : :
5946 : : /*----- Add computed group tuples to current environment. ----*/
5947 [ # # # # ]: 0 : for (auto &e : M.grouping.schema().deduplicate()) {
5948 : : try {
5949 [ # # ]: 0 : key_schema.find(e.id);
5950 [ # # ]: 0 : } catch (invalid_argument&) {
5951 : : continue; // skip duplicated keys since they must not be used afterwards
5952 [ # # ]: 0 : }
5953 : :
5954 [ # # # # ]: 0 : if (auto it = avg_aggregates.find(e.id);
5955 [ # # ]: 0 : it != avg_aggregates.end() and not it->second.compute_running_avg)
5956 : : { // AVG aggregates which is not yet computed, divide computed sum with computed count
5957 : 0 : auto &avg_info = it->second;
5958 [ # # ]: 0 : auto sum = std::visit(overloaded {
5959 : 0 : [&]<sql_type T>(HashTable::const_reference_t<T> &&r) -> _Doublex1
5960 : : requires (std::same_as<T, _I64x1> or std::same_as<T, _Doublex1>) {
5961 [ # # # # ]: 0 : return T(r).template to<double>();
5962 : 0 : },
5963 : 0 : [](auto&&) -> _Doublex1 { M_unreachable("invalid type"); },
5964 : 0 : [](std::monostate&&) -> _Doublex1 { M_unreachable("invalid reference"); },
5965 [ # # ]: 0 : }, entry.get(avg_info.sum));
5966 [ # # # # : 0 : auto count = _I64x1(entry.get<_I64x1>(avg_info.running_count)).insist_not_null().to<double>();
# # # # ]
5967 [ # # ]: 0 : auto avg = sum / count; // no need to multiply with group counter as the factor would not change the fraction
5968 [ # # ]: 0 : if (avg.can_be_null()) {
5969 [ # # ]: 0 : _Var<Doublex1> var(avg); // introduce variable s.t. uses only load from it
5970 [ # # # # : 0 : env.add(e.id, var);
# # ]
5971 : 0 : } else {
5972 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
5973 [ # # # # ]: 0 : Var<Doublex1> var(avg.insist_not_null());
5974 [ # # # # : 0 : env.add(e.id, _Doublex1(var));
# # # # ]
5975 : 0 : }
5976 : 0 : } else { // part of key or already computed aggregate (without multiplication with group counter)
5977 [ # # ]: 0 : std::visit(overloaded {
5978 : 0 : [&]<typename T>(HashTable::const_reference_t<Expr<T>> &&r) -> void {
5979 : 0 : Expr<T> value = r;
5980 : :
5981 : 0 : auto pred = [&e](const auto &info) -> bool { return info.entry.id == e.id; };
5982 [ # # # # : 0 : if (auto it = std::find_if(aggregates.cbegin(), aggregates.cend(), pred);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5983 : 0 : it != aggregates.cend())
5984 : : { // aggregate
5985 : : /* For COUNT and SUM, multiply current aggregate value with respective group counter
5986 : : * since only tuples in phase in which argument is bound are counted/summed up. */
5987 [ # # # # : 0 : if (it->args.empty()) {
# # # # #
# # # ]
5988 [ # # # # : 0 : M_insist(it->fnid == m::Function::FN_COUNT,
# # # # #
# # # ]
5989 : : "only COUNT aggregate function may have no argument");
5990 : : I64x1 probe_counter =
5991 [ # # # # : 0 : _I64x1(entry.get<_I64x1>(C.pool("$probe_counter"))).insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5992 [ # # # # : 0 : PrimitiveExpr<T> count = value.insist_not_null() * probe_counter.to<T>();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5993 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(count); // introduce variable s.t. uses only load from it
# # # # #
# # # ]
5994 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5995 : : return; // next group tuple entry
5996 : 0 : } else {
5997 [ # # # # : 0 : M_insist(it->args.size() == 1, "aggregate functions expect at most one argument");
# # # # #
# # # ]
5998 [ # # # # : 0 : auto &des = as<const Designator>(*it->args[0]);
# # # # #
# # # ]
5999 [ # # # # : 0 : Schema::Identifier arg(des.table_name.text, des.attr_name.text.assert_not_none());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6000 [ # # # # : 0 : if (it->fnid == m::Function::FN_COUNT or it->fnid == m::Function::FN_SUM) {
# # # # #
# # # # #
# # # # #
# # # #
# ]
6001 [ # # # # : 0 : if (M.probe.schema().has(arg)) {
# # # # #
# # # # #
# # # # #
# # # #
# ]
6002 : : I64x1 build_counter =
6003 [ # # # # : 0 : _I64x1(entry.get<_I64x1>(C.pool("$build_counter"))).insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
6004 [ # # # # : 0 : auto agg = value * build_counter.to<T>();
# # # # #
# # # # #
# # # # #
# # # #
# ]
6005 [ # # # # : 0 : if (agg.can_be_null()) {
# # # # #
# # # ]
6006 [ # # # # : 0 : Var<Expr<T>> var(agg); // introduce variable s.t. uses only load from it
# # # # #
# # # ]
6007 [ # # # # : 0 : env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6008 : 0 : } else {
6009 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
6010 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(agg.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # #
# ]
6011 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6012 : 0 : }
6013 : 0 : } else {
6014 [ # # # # : 0 : M_insist(M.build.schema().has(arg),
# # # # #
# # # # #
# # # # #
# # # #
# ]
6015 : : "argument ID must occur in either child schema");
6016 : : I64x1 probe_counter =
6017 [ # # # # : 0 : _I64x1(entry.get<_I64x1>(C.pool("$probe_counter"))).insist_not_null();
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
6018 [ # # # # : 0 : auto agg = value * probe_counter.to<T>();
# # # # #
# # # # #
# # # # #
# # # #
# ]
6019 [ # # # # : 0 : if (agg.can_be_null()) {
# # # # #
# # # ]
6020 [ # # # # : 0 : Var<Expr<T>> var(agg); // introduce variable s.t. uses only load from it
# # # # #
# # # ]
6021 [ # # # # : 0 : env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6022 : 0 : } else {
6023 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
6024 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(agg.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # #
# ]
6025 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6026 : 0 : }
6027 : 0 : }
6028 : 0 : return; // next group tuple entry
6029 : : }
6030 [ # # # # : 0 : }
# # # # #
# # # ]
6031 : 0 : }
6032 : :
6033 : : /* fallthrough: part of key or correctly computed aggregate */
6034 [ # # # # : 0 : if (value.can_be_null()) {
# # # # #
# # # ]
6035 [ # # # # : 0 : Var<Expr<T>> var(value); // introduce variable s.t. uses only load from it
# # # # #
# # # ]
6036 [ # # # # : 0 : env.add(e.id, var);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6037 : 0 : } else {
6038 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
6039 [ # # # # : 0 : Var<PrimitiveExpr<T>> var(value.insist_not_null());
# # # # #
# # # # #
# # # # #
# # # #
# ]
6040 [ # # # # : 0 : env.add(e.id, Expr<T>(var));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6041 : 0 : }
6042 [ # # # # : 0 : },
# # # # #
# # # ]
6043 : 0 : [&](HashTable::const_reference_t<_Boolx1> &&r) -> void {
6044 : : #ifndef NDEBUG
6045 : 0 : auto pred = [&e](const auto &info) -> bool { return info.entry.id == e.id; };
6046 : 0 : M_insist(std::find_if(aggregates.cbegin(), aggregates.cend(), pred) == aggregates.cend(),
6047 : : "booleans must not be the result of aggregate functions");
6048 : : #endif
6049 : 0 : _Boolx1 value = r;
6050 [ # # ]: 0 : if (value.can_be_null()) {
6051 [ # # ]: 0 : _Var<Boolx1> var(value); // introduce variable s.t. uses only load from it
6052 [ # # # # : 0 : env.add(e.id, var);
# # ]
6053 : 0 : } else {
6054 : : /* introduce variable w/o NULL bit s.t. uses only load from it */
6055 [ # # # # ]: 0 : Var<Boolx1> var(value.insist_not_null());
6056 [ # # # # : 0 : env.add(e.id, _Boolx1(var));
# # # # ]
6057 : 0 : }
6058 : 0 : },
6059 : 0 : [&](HashTable::const_reference_t<NChar> &&r) -> void {
6060 : : #ifndef NDEBUG
6061 : 0 : auto pred = [&e](const auto &info) -> bool { return info.entry.id == e.id; };
6062 : 0 : M_insist(std::find_if(aggregates.cbegin(), aggregates.cend(), pred) == aggregates.cend(),
6063 : : "strings must not be the result of aggregate functions");
6064 : : #endif
6065 : 0 : NChar value(r);
6066 [ # # # # ]: 0 : Var<Ptr<Charx1>> var(value.val()); // introduce variable s.t. uses only load from it
6067 [ # # # # : 0 : env.add(e.id, NChar(var, value.can_be_null(), value.length(),
# # # # #
# ]
6068 : 0 : value.guarantees_terminating_nul()));
6069 : 0 : },
6070 : 0 : [](std::monostate&&) -> void { M_unreachable("invalid reference"); },
6071 [ # # ]: 0 : }, entry.get(e.id)); // do not extract to be able to access for not-yet-computed AVG aggregates
6072 : : }
6073 : : }
6074 : :
6075 : : /*----- Resume pipeline. -----*/
6076 [ # # ]: 0 : pipeline();
6077 : 0 : };
6078 : 0 : });
6079 [ # # # # ]: 0 : teardown_t(std::move(teardown), [&](){ ht->teardown(); })();
6080 : 0 : }
6081 : :
6082 : :
6083 : : /*======================================================================================================================
6084 : : * Match<T>::print()
6085 : : *====================================================================================================================*/
6086 : :
6087 : : struct print_info
6088 : : {
6089 : : const Operator &op;
6090 : :
6091 : 0 : friend std::ostream & operator<<(std::ostream &out, const print_info &info) {
6092 [ # # ]: 0 : if (info.op.has_info())
6093 : 0 : out << " <" << info.op.info().estimated_cardinality << '>';
6094 : 0 : return out;
6095 : : }
6096 : : };
6097 : :
6098 : 0 : void Match<m::wasm::NoOp>::print(std::ostream &out, unsigned level) const
6099 : : {
6100 : 0 : indent(out, level) << "wasm::NoOp" << print_info(this->noop) << " (cumulative cost " << cost() << ')';
6101 : 0 : this->child->print(out, level + 1);
6102 : 0 : }
6103 : :
6104 : : template<bool SIMDfied>
6105 : 0 : void Match<m::wasm::Callback<SIMDfied>>::print(std::ostream &out, unsigned level) const
6106 : : {
6107 : 0 : indent(out, level) << "wasm::Callback with " << this->result_set_window_size << " tuples result set "
6108 : 0 : << this->callback.schema() << print_info(this->callback)
6109 : 0 : << " (cumulative cost " << cost() << ')';
6110 : 0 : this->child->print(out, level + 1);
6111 : 0 : }
6112 : :
6113 : : template<bool SIMDfied>
6114 : 0 : void Match<m::wasm::Print<SIMDfied>>::print(std::ostream &out, unsigned level) const
6115 : : {
6116 : 0 : indent(out, level) << "wasm::Print with " << this->result_set_window_size << " tuples result set "
6117 : 0 : << this->print_op.schema() << print_info(this->print_op)
6118 : 0 : << " (cumulative cost " << cost() << ')';
6119 : 0 : this->child->print(out, level + 1);
6120 : 0 : }
6121 : :
6122 : : template<bool SIMDfied>
6123 : 0 : void Match<m::wasm::Scan<SIMDfied>>::print(std::ostream &out, unsigned level) const
6124 : : {
6125 : 0 : indent(out, level) << (SIMDfied ? "wasm::SIMDScan(" : "wasm::Scan(") << this->scan.alias() << ") ";
6126 [ # # # # : 0 : if (this->buffer_factory_ and this->scan.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6127 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6128 : 0 : out << this->scan.schema() << print_info(this->scan) << " (cumulative cost " << cost() << ')';
6129 : 0 : }
6130 : :
6131 : : template<idx::IndexMethod IndexMethod>
6132 : 0 : void Match<m::wasm::IndexScan<IndexMethod>>::print(std::ostream &out, unsigned level) const
6133 : : {
6134 : : if (IndexMethod == idx::IndexMethod::Array)
6135 : 0 : indent(out, level) << "wasm::ArrayIndexScan(";
6136 : : else if (IndexMethod == idx::IndexMethod::Rmi)
6137 : 0 : indent(out, level) << "wasm::RecursiveModelIndexScan(";
6138 : : else
6139 : : M_unreachable("unknown index");
6140 : :
6141 [ # # # # ]: 0 : if (options::index_scan_strategy == option_configs::IndexScanStrategy::COMPILATION) {
6142 : 0 : out << "Compilation[";
6143 [ # # # # ]: 0 : if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::CALLBACK)
6144 : 0 : out << "Callback";
6145 [ # # # # ]: 0 : else if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::EXPOSED_MEMORY)
6146 : 0 : out << "ExposedMemory";
6147 : : else
6148 : 0 : M_unreachable("unknown compilation strategy");
6149 [ # # # # ]: 0 : } else if (options::index_scan_strategy == option_configs::IndexScanStrategy::INTERPRETATION) {
6150 : 0 : out << "Interpretation[";
6151 [ # # # # ]: 0 : if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::INLINE)
6152 : 0 : out << "Inline";
6153 [ # # # # ]: 0 : else if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::MEMORY)
6154 : 0 : out << "Memory";
6155 : : else
6156 : 0 : M_unreachable("unknown materialization strategy");
6157 [ # # # # ]: 0 : } else if (options::index_scan_strategy == option_configs::IndexScanStrategy::HYBRID) {
6158 : 0 : out << "Hybrid[";
6159 [ # # # # ]: 0 : if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::INLINE)
6160 : 0 : out << "Inline,";
6161 [ # # # # ]: 0 : else if (options::index_scan_materialization_strategy == option_configs::IndexScanMaterializationStrategy::MEMORY)
6162 : 0 : out << "Memory,";
6163 : : else
6164 : 0 : M_unreachable("unknown materialization strategy");
6165 [ # # # # ]: 0 : if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::CALLBACK)
6166 : 0 : out << "Callback";
6167 [ # # # # ]: 0 : else if (options::index_scan_compilation_strategy == option_configs::IndexScanCompilationStrategy::EXPOSED_MEMORY)
6168 : 0 : out << "ExposedMemory";
6169 : : else
6170 : 0 : M_unreachable("unknown compilation strategy");
6171 : 0 : } else {
6172 : 0 : M_unreachable("unknown strategy");
6173 : : }
6174 : :
6175 : 0 : out << "], " << this->scan.alias() << ", " << this->filter.filter() << ") ";
6176 [ # # # # : 0 : if (this->buffer_factory_ and this->scan.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6177 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6178 : 0 : out << this->scan.schema() << print_info(this->scan) << " (cumulative cost " << cost() << ')';
6179 : 0 : }
6180 : :
6181 : : template<bool Predicated>
6182 : 0 : void Match<m::wasm::Filter<Predicated>>::print(std::ostream &out, unsigned level) const
6183 : : {
6184 : 0 : indent(out, level) << "wasm::" << (Predicated ? "Predicated" : "Branching") << "Filter ";
6185 [ # # # # : 0 : if (this->buffer_factory_ and this->filter.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6186 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6187 : 0 : out << this->filter.schema() << print_info(this->filter) << " (cumulative cost " << cost() << ')';
6188 : 0 : this->child->print(out, level + 1);
6189 : 0 : }
6190 : :
6191 : 0 : void Match<m::wasm::LazyDisjunctiveFilter>::print(std::ostream &out, unsigned level) const
6192 : : {
6193 : 0 : indent(out, level) << "wasm::LazyDisjunctiveFilter ";
6194 [ # # # # : 0 : if (this->buffer_factory_ and this->filter.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # ]
6195 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6196 : 0 : const cnf::Clause &clause = this->filter.filter()[0];
6197 [ # # ]: 0 : for (auto it = clause.cbegin(); it != clause.cend(); ++it) {
6198 [ # # ]: 0 : if (it != clause.cbegin()) out << " → ";
6199 : 0 : out << *it;
6200 : 0 : }
6201 : 0 : out << ' ' << this->filter.schema() << print_info(this->filter) << " (cumulative cost " << cost() << ')';
6202 : 0 : this->child->print(out, level + 1);
6203 : 0 : }
6204 : :
6205 : 0 : void Match<m::wasm::Projection>::print(std::ostream &out, unsigned level) const
6206 : : {
6207 : 0 : indent(out, level) << "wasm::Projection ";
6208 [ # # # # : 0 : if (this->buffer_factory_ and this->projection.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # ]
6209 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6210 : 0 : out << this->projection.schema() << print_info(this->projection) << " (cumulative cost " << cost() << ')';
6211 [ # # ]: 0 : if (this->child)
6212 : 0 : this->child->get()->print(out, level + 1);
6213 : 0 : }
6214 : :
6215 : 0 : void Match<m::wasm::HashBasedGrouping>::print(std::ostream &out, unsigned level) const
6216 : : {
6217 : 0 : indent(out, level) << "wasm::HashBasedGrouping " << this->grouping.schema() << print_info(this->grouping)
6218 : 0 : << " (cumulative cost " << cost() << ')';
6219 : 0 : this->child->print(out, level + 1);
6220 : 0 : }
6221 : :
6222 : 0 : void Match<m::wasm::OrderedGrouping>::print(std::ostream &out, unsigned level) const
6223 : : {
6224 : 0 : indent(out, level) << "wasm::OrderedGrouping " << this->grouping.schema() << print_info(this->grouping)
6225 : 0 : << " (cumulative cost " << cost() << ')';
6226 : 0 : this->child->print(out, level + 1);
6227 : 0 : }
6228 : :
6229 : 0 : void Match<m::wasm::Aggregation>::print(std::ostream &out, unsigned level) const
6230 : : {
6231 : 0 : indent(out, level) << "wasm::Aggregation " << this->aggregation.schema() << print_info(this->aggregation)
6232 : 0 : << " (cumulative cost " << cost() << ')';
6233 : 0 : this->child->print(out, level + 1);
6234 : 0 : }
6235 : :
6236 : : template<bool CmpPredicated>
6237 : 0 : void Match<m::wasm::Quicksort<CmpPredicated>>::print(std::ostream &out, unsigned level) const
6238 : : {
6239 : 0 : indent(out, level) << "wasm::" << (CmpPredicated ? "Predicated" : "") << "Quicksort " << this->sorting.schema()
6240 : 0 : << print_info(this->sorting) << " (cumulative cost " << cost() << ')';
6241 : 0 : this->child->print(out, level + 1);
6242 : 0 : }
6243 : :
6244 : 0 : void Match<m::wasm::NoOpSorting>::print(std::ostream &out, unsigned level) const
6245 : : {
6246 : 0 : indent(out, level) << "wasm::NoOpSorting" << print_info(this->sorting) << " (cumulative cost " << cost() << ')';
6247 : 0 : this->child->print(out, level + 1);
6248 : 0 : }
6249 : :
6250 : : template<bool Predicated>
6251 : 0 : void Match<m::wasm::NestedLoopsJoin<Predicated>>::print(std::ostream &out, unsigned level) const
6252 : : {
6253 : 0 : indent(out, level) << "wasm::" << (Predicated ? "Predicated" : "") << "NestedLoopsJoin ";
6254 [ # # # # : 0 : if (this->buffer_factory_ and this->join.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6255 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6256 : 0 : out << this->join.schema() << print_info(this->join) << " (cumulative cost " << cost() << ')';
6257 : :
6258 : 0 : ++level;
6259 : 0 : std::size_t i = this->children.size();
6260 [ # # # # ]: 0 : while (i--) {
6261 : 0 : const m::wasm::MatchBase &child = *this->children[i];
6262 : 0 : indent(out, level) << i << ". input";
6263 : 0 : child.print(out, level + 1);
6264 : : }
6265 : 0 : }
6266 : :
6267 : : template<bool Unique, bool Predicated>
6268 : 0 : void Match<m::wasm::SimpleHashJoin<Unique, Predicated>>::print(std::ostream &out, unsigned level) const
6269 : : {
6270 : 0 : indent(out, level) << "wasm::" << (Predicated ? "Predicated" : "") << "SimpleHashJoin";
6271 : 0 : if (Unique) out << " on UNIQUE key ";
6272 [ # # # # : 0 : if (this->buffer_factory_ and this->join.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
6273 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6274 : 0 : out << this->join.schema() << print_info(this->join) << " (cumulative cost " << cost() << ')';
6275 : :
6276 : 0 : ++level;
6277 : 0 : const m::wasm::MatchBase &build = *this->children[0];
6278 : 0 : const m::wasm::MatchBase &probe = *this->children[1];
6279 : 0 : indent(out, level) << "probe input";
6280 : 0 : probe.print(out, level + 1);
6281 : 0 : indent(out, level) << "build input";
6282 : 0 : build.print(out, level + 1);
6283 : 0 : }
6284 : :
6285 : : template<bool SortLeft, bool SortRight, bool Predicated, bool CmpPredicated>
6286 : 0 : void Match<m::wasm::SortMergeJoin<SortLeft, SortRight, Predicated, CmpPredicated>>::print(std::ostream &out,
6287 : : unsigned level) const
6288 : : {
6289 : 0 : indent(out, level) << "wasm::" << (Predicated ? "Predicated" : "") << "SortMergeJoin ";
6290 : : switch ((unsigned(SortLeft) << 1) | unsigned(SortRight))
6291 : : {
6292 : 0 : case 0: out << "pre-sorted "; break;
6293 : 0 : case 1: out << "sorting right input " << (CmpPredicated ? "predicated " : ""); break;
6294 : 0 : case 2: out << "sorting left input " << (CmpPredicated ? "predicated " : ""); break;
6295 : 0 : case 3: out << "sorting both inputs " << (CmpPredicated ? "predicated " : ""); break;
6296 : : }
6297 [ # # # # : 0 : const bool needs_buffer_parent = not is<const ScanOperator>(this->parent) or SortLeft;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6298 [ # # # # : 0 : const bool needs_buffer_child = not is<const ScanOperator>(this->child) or SortRight;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6299 [ # # # # : 0 : if (needs_buffer_parent and needs_buffer_child)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
6300 : 0 : out << "and materializing both inputs ";
6301 [ # # # # : 0 : else if (needs_buffer_parent)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6302 : 0 : out << "and materializing left input ";
6303 [ # # # # : 0 : else if (needs_buffer_child)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6304 : 0 : out << "and materializing right input ";
6305 : 0 : out << this->join.schema() << print_info(this->join) << " (cumulative cost " << cost() << ')';
6306 : :
6307 : 0 : ++level;
6308 : 0 : const m::wasm::MatchBase &left = *this->children[0];
6309 : 0 : const m::wasm::MatchBase &right = *this->children[1];
6310 : 0 : indent(out, level) << "right input";
6311 : 0 : right.print(out, level + 1);
6312 : 0 : indent(out, level) << "left input";
6313 : 0 : left.print(out, level + 1);
6314 : 0 : }
6315 : :
6316 : 0 : void Match<m::wasm::Limit>::print(std::ostream &out, unsigned level) const
6317 : : {
6318 : 0 : indent(out, level) << "wasm::Limit " << this->limit.schema() << print_info(this->limit)
6319 : 0 : << " (cumulative cost " << cost() << ')';
6320 : 0 : this->child->print(out, level + 1);
6321 : 0 : }
6322 : :
6323 : 0 : void Match<m::wasm::HashBasedGroupJoin>::print(std::ostream &out, unsigned level) const
6324 : : {
6325 : 0 : indent(out, level) << "wasm::HashBasedGroupJoin ";
6326 [ # # # # : 0 : if (this->buffer_factory_ and this->grouping.schema().drop_constants().deduplicate().num_entries())
# # # # #
# # # # #
# # ]
6327 : 0 : out << "with " << this->buffer_num_tuples_ << " tuples output buffer ";
6328 : 0 : out << this->grouping.schema() << print_info(this->grouping) << " (cumulative cost " << cost() << ')';
6329 : :
6330 : 0 : ++level;
6331 : 0 : const m::wasm::MatchBase &build = *this->children[0];
6332 : 0 : const m::wasm::MatchBase &probe = *this->children[1];
6333 : 0 : indent(out, level) << "probe input";
6334 : 0 : probe.print(out, level + 1);
6335 : 0 : indent(out, level) << "build input";
6336 : 0 : build.print(out, level + 1);
6337 : 0 : }
6338 : :
6339 : :
6340 : : /*======================================================================================================================
6341 : : * ThePreOrderMatchBaseVisitor, ThePostOrderMatchBaseVisitor
6342 : : *====================================================================================================================*/
6343 : :
6344 : : namespace {
6345 : :
6346 : : template<bool C, bool PreOrder>
6347 : : struct recursive_matchbase_visitor : TheRecursiveMatchBaseVisitorBase<C>
6348 : : {
6349 : : using super = TheRecursiveMatchBaseVisitorBase<C>;
6350 : : template<typename T> using Const = typename super::template Const<T>;
6351 : : using callback_t = std::conditional_t<C, ConstMatchBaseVisitor, MatchBaseVisitor>;
6352 : :
6353 : : private:
6354 : : callback_t &callback_;
6355 : :
6356 : : public:
6357 : 0 : recursive_matchbase_visitor(callback_t &callback) : callback_(callback) { }
6358 : :
6359 : : using super::operator();
6360 : : #define DECLARE(CLASS) \
6361 : : void operator()(Const<CLASS> &M) override { \
6362 : : if constexpr (PreOrder) try { callback_(M); } catch (visit_skip_subtree) { return; } \
6363 : : super::operator()(M); \
6364 : : if constexpr (not PreOrder) callback_(M); \
6365 : : }
6366 [ # # # # : 0 : M_WASM_MATCH_LIST(DECLARE)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6367 : : #undef DECLARE
6368 : : };
6369 : :
6370 : : }
6371 : :
6372 : : template<bool C>
6373 : 0 : void ThePreOrderMatchBaseVisitor<C>::operator()(Const<MatchBase> &M)
6374 : : {
6375 [ # # ]: 0 : recursive_matchbase_visitor<C, /* PreOrder= */ true>{*this}(M);
6376 : 0 : }
6377 : :
6378 : : template<bool C>
6379 : 0 : void ThePostOrderMatchBaseVisitor<C>::operator()(Const<MatchBase> &M)
6380 : : {
6381 [ # # ]: 0 : recursive_matchbase_visitor<C, /* PreOrder= */ false>{*this}(M);
6382 : 0 : }
6383 : :
6384 : :
6385 : : /*======================================================================================================================
6386 : : * Explicit template instantiations
6387 : : *====================================================================================================================*/
6388 : :
6389 : : #define INSTANTIATE(CLASS) \
6390 : : template struct m::wasm::CLASS; \
6391 : : template struct m::Match<m::wasm::CLASS>;
6392 : : M_WASM_OPERATOR_LIST_TEMPLATED(INSTANTIATE)
6393 : : #undef INSTANTIATE
6394 : :
6395 : : template struct m::wasm::ThePreOrderMatchBaseVisitor<true>;
6396 : : template struct m::wasm::ThePostOrderMatchBaseVisitor<true>;
|
