BitMagic-C++: bmfunc.h Source File

BitMagic-C++

bmfunc.h
Go to the documentation of this file.
1 #ifndef BMFUNC__H__INCLUDED__
2 #define BMFUNC__H__INCLUDED__
3 /*
4 Copyright(c) 2002-2017 Anatoliy Kuznetsov(anatoliy_kuznetsov at yahoo.com)
5
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
9
10 http://www.apache.org/licenses/LICENSE-2.0
11
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17
18 For more information please visit: http://bitmagic.io
19 */
20
21 /*! \file bmfunc.h
22 \brief Bit manipulation primitives (internal)
23 */
24
25 #include <memory.h>
26
27 #include "bmdef.h"
28 #include "bmutil.h"
29
30
31 #ifdef _MSC_VER
32 # pragma warning( disable: 4146 )
33 #endif
34
35 namespace bm
36 {
37
38
39 inline
40 bm::id_t bit_block_calc_count_range(const bm::word_t* block,
41 bm::word_t left,
42 bm::word_t right) BMNOEXCEPT;
43
44 inline
45 bm::id_t bit_block_any_range(const bm::word_t* block,
46 bm::word_t left,
47 bm::word_t right) BMNOEXCEPT;
48
49 /*!
50 @brief Structure with statistical information about memory
51 allocation footprint, serialization projection, number of vectors
52 @ingroup bvector
53 */
54 struct bv_statistics
55 {
56 size_t bit_blocks; ///< Number of bit blocks
57 size_t gap_blocks; ///< Number of GAP blocks
58 size_t ptr_sub_blocks; ///< Number of sub-blocks
59 size_t bv_count; ///< Number of bit-vectors
60 size_t max_serialize_mem; ///< estimated maximum memory for serialization
61 size_t memory_used; ///< memory usage for all blocks and service tables
62 size_t gap_cap_overhead; ///< gap memory overhead between length and capacity
63 gap_word_t gap_levels[bm::gap_levels]; ///< GAP block lengths in the bvect
64 unsigned long long gaps_by_level[bm::gap_levels]; ///< number of GAP blocks at each level
65
66 /// cound bit block
67 void add_bit_block() BMNOEXCEPT
68 {
69 ++bit_blocks;
70 size_t mem_used = sizeof(bm::word_t) * bm::set_block_size;
71 memory_used += mem_used;
72 max_serialize_mem += mem_used;
73 }
74
75 /// count gap block
76 void add_gap_block(unsigned capacity, unsigned length) BMNOEXCEPT
77 {
78 ++gap_blocks;
79 size_t mem_used = (capacity * sizeof(gap_word_t));
80 memory_used += mem_used;
81 max_serialize_mem += (unsigned)(length * sizeof(gap_word_t));
82 BM_ASSERT(length <= capacity);
83 gap_cap_overhead += (capacity - length) * sizeof(gap_word_t);
84 for (unsigned i = 0; i < bm::gap_levels; ++i)
85 {
86 if (capacity == gap_levels[i])
87 {
88 gaps_by_level[i]++;
89 return;
90 }
91 }
92 BM_ASSERT(0);
93 }
94
95 /// Reset statisctics
96 void reset() BMNOEXCEPT
97 {
98 bit_blocks = gap_blocks = ptr_sub_blocks = bv_count = 0;
99 max_serialize_mem = memory_used = gap_cap_overhead = 0;
100 for (unsigned i = 0; i < bm::gap_levels; ++i)
101 gaps_by_level[i] = 0ull;
102 }
103
104 /// Sum data from another sttructure
105 void add(const bv_statistics& st) BMNOEXCEPT
106 {
107 bit_blocks += st.bit_blocks;
108 gap_blocks += st.gap_blocks;
109 ptr_sub_blocks += st.ptr_sub_blocks;
110 bv_count += st.bv_count;
111 max_serialize_mem += st.max_serialize_mem + 8;
112 memory_used += st.memory_used;
113 gap_cap_overhead += st.gap_cap_overhead;
114 }
115 };
116
117 /**
118 @brief Pair type
119 */
120 template<typename First, typename Second>
121 struct pair
122 {
123 First first;
124 Second second;
125
126 pair(First f, Second s) : first(f), second(s) {}
127 };
128
129 /**
130 \brief bit-decode cache structure
131 */
132 struct bit_decode_cache
133 {
134 unsigned short bits[65]; //< decoded bits
135 unsigned bcnt; //< length of bits array
136 bm::id64_t cvalue; //< cache decoded value
137
138 bit_decode_cache() : bcnt(0), cvalue(0) {}
139 };
140
141
142 /**
143 \brief Recalc linear bvector block index into 2D matrix coordinates
144 \internal
145 */
146 template<typename BI_TYPE>
147 void get_block_coord(BI_TYPE nb, unsigned& i, unsigned& j) BMNOEXCEPT
148 {
149 i = unsigned(nb >> bm::set_array_shift); // top block address
150 j = unsigned(nb & bm::set_array_mask); // address in sub-block
151 }
152
153 /**
154 Compute bit address of the first bit in a superblock
155 \internal
156 */
157 template<typename RTYPE>
158 BMFORCEINLINE RTYPE get_super_block_start(unsigned i) BMNOEXCEPT
159 {
160 return RTYPE(i) * bm::set_sub_total_bits;
161 }
162
163 /**
164 Compute bit address of the first bit in a block
165 \internal
166 */
167 template<typename RTYPE>
168 BMFORCEINLINE RTYPE get_block_start(unsigned i, unsigned j) BMNOEXCEPT
169 {
170 RTYPE idx = bm::get_super_block_start<RTYPE>(i);
171 idx += (j) * bm::gap_max_bits;
172 return idx;
173 }
174
175 /*!
176 @defgroup gapfunc GAP functions
177 GAP functions implement different opereations on GAP compressed blocks (internals)
178 and serve as a minimal building blocks.
179 \internal
180 @ingroup bvector
181 */
182
183 /*!
184 @defgroup bitfunc BIT functions
185 Bit functions implement different opereations on bit blocks (internals)
186 and serve as a minimal building blocks.
187 \internal
188 @ingroup bvector
189 */
190
191
192 /*!
193 Returns bit count
194 @ingroup bitfunc
195 */
196 BMFORCEINLINE
197 bm::id_t word_bitcount(bm::id_t w) BMNOEXCEPT
198 {
199 #if defined(BMSSE42OPT) || defined(BMAVX2OPT)
200 return bm::id_t(_mm_popcnt_u32(w));
201 #else
202 #if defined(BM_USE_GCC_BUILD)
203 return (bm::id_t)__builtin_popcount(w);
204 #else
205 return
206 bm::bit_count_table<true>::_count[(unsigned char)(w)] +
207 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 8)] +
208 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 16)] +
209 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 24)];
210 #endif
211 #endif
212 }
213
214 inline
215 int parallel_popcnt_32(unsigned int n) BMNOEXCEPT
216 {
217 unsigned int tmp;
218
219 tmp = n - ((n >> 1) & 033333333333)
220 - ((n >> 2) & 011111111111);
221 return ((tmp + (tmp >> 3)) & 030707070707) % 63;
222 }
223
224
225 /*!
226 Function calculates number of 1 bits in 64-bit word.
227 @ingroup bitfunc
228 */
229 BMFORCEINLINE
230 unsigned word_bitcount64(bm::id64_t x) BMNOEXCEPT
231 {
232 #if defined(BMSSE42OPT) || defined(BMAVX2OPT)
233 #if defined(BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
234 return unsigned(_mm_popcnt_u64(x));
235 #else
236 return _mm_popcnt_u32(x >> 32) + _mm_popcnt_u32((unsigned)x);
237 #endif
238 #else
239 #if defined(BM_USE_GCC_BUILD)
240 return (unsigned)__builtin_popcountll(x);
241 #else
242 x = x - ((x >> 1) & 0x5555555555555555);
243 x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);
244 x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0F;
245 x = x + (x >> 8);
246 x = x + (x >> 16);
247 x = x + (x >> 32);
248 return x & 0xFF;
249 #endif
250 #endif
251 }
252
253 inline
254 unsigned bitcount64_4way(bm::id64_t x, bm::id64_t y,
255 bm::id64_t u, bm::id64_t v) BMNOEXCEPT
256 {
257 const bm::id64_t m1 = 0x5555555555555555U;
258 const bm::id64_t m2 = 0x3333333333333333U;
259 const bm::id64_t m3 = 0x0F0F0F0F0F0F0F0FU;
260 const bm::id64_t m4 = 0x000000FF000000FFU;
261
262 x = x - ((x >> 1) & m1);
263 y = y - ((y >> 1) & m1);
264 u = u - ((u >> 1) & m1);
265 v = v - ((v >> 1) & m1);
266 x = (x & m2) + ((x >> 2) & m2);
267 y = (y & m2) + ((y >> 2) & m2);
268 u = (u & m2) + ((u >> 2) & m2);
269 v = (v & m2) + ((v >> 2) & m2);
270 x = x + y;
271 u = u + v;
272 x = (x & m3) + ((x >> 4) & m3);
273 u = (u & m3) + ((u >> 4) & m3);
274 x = x + u;
275 x = x + (x >> 8);
276 x = x + (x >> 16);
277 x = x & m4;
278 x = x + (x >> 32);
279 return x & 0x000001FFU;
280 }
281
282
283 // --------------------------------------------------------------
284 // Functions for bit-scanning
285 // --------------------------------------------------------------
286
287 /*!
288 \brief Templated algorithm to unpacks octet based word into list of ON bit indexes
289 \param w - value
290 \param func - bit functor
291
292 @ingroup bitfunc
293 */
294 template<typename T, typename F>
295 void bit_for_each_4(T w, F& func)
296 {
297 for (unsigned sub_octet = 0; w != 0; w >>= 4, sub_octet += 4)
298 {
299 switch (w & 15) // 1111
300 {
301 case 0: // 0000
302 break;
303 case 1: // 0001
304 func(sub_octet);
305 break;
306 case 2: // 0010
307 func(sub_octet + 1);
308 break;
309 case 3: // 0011
310 func(sub_octet, sub_octet + 1);
311 break;
312 case 4: // 0100
313 func(sub_octet + 2);
314 break;
315 case 5: // 0101
316 func(sub_octet, sub_octet + 2);
317 break;
318 case 6: // 0110
319 func(sub_octet + 1, sub_octet + 2);
320 break;
321 case 7: // 0111
322 func(sub_octet, sub_octet + 1, sub_octet + 2);
323 break;
324 case 8: // 1000
325 func(sub_octet + 3);
326 break;
327 case 9: // 1001
328 func(sub_octet, sub_octet + 3);
329 break;
330 case 10: // 1010
331 func(sub_octet + 1, sub_octet + 3);
332 break;
333 case 11: // 1011
334 func(sub_octet, sub_octet + 1, sub_octet + 3);
335 break;
336 case 12: // 1100
337 func(sub_octet + 2, sub_octet + 3);
338 break;
339 case 13: // 1101
340 func(sub_octet, sub_octet + 2, sub_octet + 3);
341 break;
342 case 14: // 1110
343 func(sub_octet + 1, sub_octet + 2, sub_octet + 3);
344 break;
345 case 15: // 1111
346 func(sub_octet, sub_octet + 1, sub_octet + 2, sub_octet + 3);
347 break;
348 default:
349 BM_ASSERT(0);
350 break;
351 }
352
353 } // for
354 }
355
356
357 /*!
358 \brief Templated algorithm to unpacks word into list of ON bit indexes
359 \param w - value
360 \param func - bit functor
361
362 @ingroup bitfunc
363 */
364 template<typename T, typename F>
365 void bit_for_each(T w, F& func)
366 {
367 // Note: 4-bit table method works slower than plain check approach
368 for (unsigned octet = 0; w != 0; w >>= 8, octet += 8)
369 {
370 if (w & 1) func(octet + 0);
371 if (w & 2) func(octet + 1);
372 if (w & 4) func(octet + 2);
373 if (w & 8) func(octet + 3);
374 if (w & 16) func(octet + 4);
375 if (w & 32) func(octet + 5);
376 if (w & 64) func(octet + 6);
377 if (w & 128) func(octet + 7);
378
379 } // for
380 }
381
382 /*! @brief Adaptor to copy 1 bits to array
383 @internal
384 */
385 template<typename B>
386 class copy_to_array_functor
387 {
388 public:
389 copy_to_array_functor(B* bits): bp_(bits)
390 {}
391
392 B* ptr() { return bp_; }
393
394 void operator()(unsigned bit_idx) BMNOEXCEPT { *bp_++ = (B)bit_idx; }
395
396 void operator()(unsigned bit_idx0,
397 unsigned bit_idx1) BMNOEXCEPT
398 {
399 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
400 bp_+=2;
401 }
402
403 void operator()(unsigned bit_idx0,
404 unsigned bit_idx1,
405 unsigned bit_idx2) BMNOEXCEPT
406 {
407 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1; bp_[2] = (B)bit_idx2;
408 bp_+=3;
409 }
410
411 void operator()(unsigned bit_idx0,
412 unsigned bit_idx1,
413 unsigned bit_idx2,
414 unsigned bit_idx3) BMNOEXCEPT
415 {
416 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
417 bp_[2] = (B)bit_idx2; bp_[3] = (B)bit_idx3;
418 bp_+=4;
419 }
420
421 private:
422 copy_to_array_functor(const copy_to_array_functor&);
423 copy_to_array_functor& operator=(const copy_to_array_functor&);
424 private:
425 B* bp_;
426 };
427
428
429 /*!
430 \brief Unpacks word into list of ON bit indexes
431 \param w - value
432 \param bits - pointer on the result array
433 \return number of bits in the list
434
435 @ingroup bitfunc
436 */
437 template<typename T,typename B>
438 unsigned bit_list(T w, B* bits) BMNOEXCEPT
439 {
440 copy_to_array_functor<B> func(bits);
441 bit_for_each(w, func);
442 return (unsigned)(func.ptr() - bits);
443 }
444
445
446
447 /*!
448 \brief Unpacks word into list of ON bit indexes (quad-bit based)
449 \param w - value
450 \param bits - pointer on the result array
451 \return number of bits in the list
452
453 @ingroup bitfunc
454 */
455 template<typename T,typename B>
456 unsigned bit_list_4(T w, B* bits) BMNOEXCEPT
457 {
458 copy_to_array_functor<B> func(bits);
459 bit_for_each_4(w, func);
460 return (unsigned)(func.ptr() - bits);
461 }
462
463 /*!
464 \brief Unpacks word into list of ON bit indexes using popcnt method
465 \param w - value
466 \param bits - pointer on the result array
467 \param offs - value to add to bit position (programmed shift)
468 \return number of bits in the list
469
470 @ingroup bitfunc
471 @internal
472 */
473 template<typename B>
474 unsigned short
475 bitscan_popcnt(bm::id_t w, B* bits, unsigned short offs) BMNOEXCEPT
476 {
477 unsigned pos = 0;
478 while (w)
479 {
480 bm::id_t t = w & -w;
481 bits[pos++] = (B)(bm::word_bitcount(t - 1) + offs);
482 w &= w - 1;
483 }
484 return (unsigned short)pos;
485 }
486
487 /*!
488 \brief Unpacks word into list of ON bit indexes using popcnt method
489 \param w - value
490 \param bits - pointer on the result array
491 \return number of bits in the list
492
493 @ingroup bitfunc
494 @internal
495 */
496 template<typename B>
497 unsigned short bitscan_popcnt(bm::id_t w, B* bits) BMNOEXCEPT
498 {
499 unsigned pos = 0;
500 while (w)
501 {
502 bm::id_t t = w & -w;
503 bits[pos++] = (B)(bm::word_bitcount(t - 1));
504 w &= w - 1;
505 }
506 return (unsigned short)pos;
507 }
508
509
510 /*!
511 \brief Unpacks 64-bit word into list of ON bit indexes using popcnt method
512 \param w - value
513 \param bits - pointer on the result array
514 \return number of bits in the list
515 @ingroup bitfunc
516 */
517 template<typename B>
518 unsigned short bitscan_popcnt64(bm::id64_t w, B* bits) BMNOEXCEPT
519 {
520 unsigned short pos = 0;
521 while (w)
522 {
523 bm::id64_t t = bmi_blsi_u64(w); // w & -w;
524 bits[pos++] = (B) bm::word_bitcount64(t - 1);
525 w = bmi_bslr_u64(w); // w &= w - 1;
526 }
527 return pos;
528 }
529
530 /*!
531 \brief Unpacks 64-bit word into list of ON bit indexes using popcnt method
532 \param w - value
533 \param bits - pointer on the result array
534 \param offs - value to add to bit position (programmed shift)
535 \return number of bits in the list
536 @ingroup bitfunc
537 */
538 template<typename B>
539 unsigned short
540 bitscan_popcnt64(bm::id64_t w, B* bits, unsigned short offs) BMNOEXCEPT
541 {
542 unsigned short pos = 0;
543 while (w)
544 {
545 bm::id64_t t = bmi_blsi_u64(w); // w & -w;
546 bits[pos++] = B(bm::word_bitcount64(t - 1) + offs);
547 w = bmi_bslr_u64(w); // w &= w - 1;
548 }
549 return pos;
550 }
551
552
553 template<typename V, typename B>
554 unsigned short bitscan(V w, B* bits) BMNOEXCEPT
555 {
556 if (bm::conditional<sizeof(V) == 8>::test())
557 return bm::bitscan_popcnt64(w, bits);
558 else
559 return bm::bitscan_popcnt((bm::word_t)w, bits);
560 }
561
562 // --------------------------------------------------------------
563 // Functions for select
564 // --------------------------------------------------------------
565
566 /**
567 \brief word find index of the rank-th bit set by bit-testing
568 \param w - 64-bit work to search
569 \param rank - rank to select (should be > 0)
570
571 \return selected value (inxed of bit set)
572 */
573 inline
574 unsigned word_select64_linear(bm::id64_t w, unsigned rank) BMNOEXCEPT
575 {
576 BM_ASSERT(w);
577 BM_ASSERT(rank);
578
579 for (unsigned count = 0; w; w >>=1ull, ++count)
580 {
581 rank -= unsigned(w & 1ull);
582 if (!rank)
583 return count;
584 }
585 BM_ASSERT(0); // shoud not be here if rank is achievable
586 return ~0u;
587 }
588
589 /**
590 \brief word find index of the rank-th bit set by bit-testing
591 \param w - 64-bit work to search
592 \param rank - rank to select (should be > 0)
593
594 \return selected value (inxed of bit set)
595 */
596 inline
597 unsigned word_select64_bitscan(bm::id64_t w, unsigned rank) BMNOEXCEPT
598 {
599 BM_ASSERT(w);
600 BM_ASSERT(rank);
601 BM_ASSERT(rank <= bm::word_bitcount64(w));
602
603 do
604 {
605 --rank;
606 if (!rank)
607 break;
608 w &= w - 1;
609 } while (1);
610
611 bm::id64_t t = w & -w;
612 unsigned count = bm::word_bitcount64(t - 1);
613 return count;
614 }
615
616 /**
617 \brief word find index of the rank-th bit set by bit-testing
618 \param w - 64-bit work to search
619 \param rank - rank to select (should be > 0)
620
621 \return selected value (inxed of bit set)
622 */
623 inline
624 unsigned word_select64(bm::id64_t w, unsigned rank) BMNOEXCEPT
625 {
626 #if defined(BMI2_SELECT64)
627 return BMI2_SELECT64(w, rank);
628 #else
629 #if defined(BMI1_SELECT64)
630 return BMI2_SELECT64(w, rank);
631 #else
632 return bm::word_select64_bitscan(w, rank);
633 #endif
634 #endif
635 }
636
637 // --------------------------------------------------------------
638 // Functions for bit-block digest calculation
639 // --------------------------------------------------------------
640
641 /*!
642 \brief Compute digest mask for word address in block
643
644 @return digest bit-mask
645
646 @ingroup bitfunc
647 @internal
648 */
649 BMFORCEINLINE
650 bm::id64_t widx_to_digest_mask(unsigned w_idx) BMNOEXCEPT
651 {
652 bm::id64_t mask(1ull);
653 return mask << (w_idx / bm::set_block_digest_wave_size);
654 }
655
656 /**
657 \brief Compute digest mask for [from..to] positions
658 \param from - range from (in bit-block coordinates)
659 \param to - range to (in bit-block coordinates)
660
661 @ingroup bitfunc
662 @internal
663 */
664 BMFORCEINLINE
665 bm::id64_t digest_mask(unsigned from, unsigned to) BMNOEXCEPT
666 {
667 BM_ASSERT(from <= to);
668
669 bm::id64_t digest_from = from >> bm::set_block_digest_pos_shift;
670 bm::id64_t digest_to = to >> bm::set_block_digest_pos_shift;;
671 const bm::id64_t maskFF(~0ull);
672 return ((maskFF) >> (63 - (digest_to - digest_from))) << digest_from;
673 }
674
675 /*!
676 \brief check if all digest bits for the range [from..to] are 0
677
678 \param digest - current digest
679 \param bitpos_from - range from (in bit-block coordinates)
680 \param bitpos_to - range to (in bit-block coordinates)
681
682 @return true if all range is zero
683
684 @ingroup bitfunc
685 @internal
686 */
687 inline
688 bool check_zero_digest(bm::id64_t digest,
689 unsigned bitpos_from, unsigned bitpos_to) BMNOEXCEPT
690 {
691 bm::id64_t mask = bm::digest_mask(bitpos_from, bitpos_to);
692 return !(digest & mask);
693 }
694
695 /*!
696 \brief Init block with 000111000 pattren based on digest
697 \param block - Bit block [out]
698 \param digest - digest (used for block initialization)
699
700 @ingroup bitfunc
701 @internal
702 */
703 inline
704 void block_init_digest0(bm::word_t* const block, bm::id64_t digest) BMNOEXCEPT
705 {
706 unsigned off;
707 for (unsigned i = 0; i < 64; ++i)
708 {
709 off = i * bm::set_block_digest_wave_size;
710 bm::word_t mask = (digest & 1) ? ~0u : 0u;
711 #if defined(VECT_BLOCK_SET_DIGEST)
712 VECT_BLOCK_SET_DIGEST(&block[off], mask);
713 #else
714 for (unsigned j = 0; j < bm::set_block_digest_wave_size; j+=4)
715 {
716 block[off+j+0] = block[off+j+1] =
717 block[off+j+2] = block[off+j+3] = mask;
718 } // for j
719 #endif
720 digest >>= 1ull;
721 } // for
722 }
723
724 /*!
725 \brief Compute digest for 64 non-zero areas
726 \param block - Bit block
727
728 @return digest bit-mask (0 means all block is empty)
729
730 @ingroup bitfunc
731 @internal
732 */
733 inline
734 bm::id64_t calc_block_digest0(const bm::word_t* const block) BMNOEXCEPT
735 {
736 bm::id64_t digest0 = 0;
737 unsigned off;
738
739 for (unsigned i = 0; i < 64; ++i)
740 {
741 off = i * bm::set_block_digest_wave_size;
742 #if defined(VECT_IS_DIGEST_ZERO)
743 bool all_zero = VECT_IS_DIGEST_ZERO(&block[off]);
744 digest0 |= bm::id64_t(!all_zero) << i;
745 #else
746 const bm::id64_t mask(1ull);
747 for (unsigned j = 0; j < bm::set_block_digest_wave_size; j+=4)
748 {
749 bm::word_t w =
750 block[off+j+0] | block[off+j+1] | block[off+j+2] | block[off+j+3];
751 if (w)
752 {
753 digest0 |= (mask << i);
754 break;
755 }
756 } // for j
757 #endif
758
759 } // for i
760 return digest0;
761 }
762
763 /*!
764 \brief Compute digest for 64 non-zero areas based on existing digest
765 (function revalidates zero areas)
766 \param block - bit block
767 \param digest - start digest
768
769 @return digest bit-mask (0 means all block is empty)
770
771 @ingroup bitfunc
772 @internal
773 */
774 inline
775 bm::id64_t
776 update_block_digest0(const bm::word_t* const block, bm::id64_t digest) BMNOEXCEPT
777 {
778 const bm::id64_t mask(1ull);
779 bm::id64_t d = digest;
780
781 while (d)
782 {
783 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
784
785 unsigned wave = bm::word_bitcount64(t - 1);
786 unsigned off = wave * bm::set_block_digest_wave_size;
787
788 #if defined(VECT_IS_DIGEST_ZERO)
789 bool all_zero = VECT_IS_DIGEST_ZERO(&block[off]);
790 digest &= all_zero ? ~(mask << wave) : digest;
791 #else
792 const bm::bit_block_t::bunion_t* BMRESTRICT src_u =
793 (const bm::bit_block_t::bunion_t*)(&block[off]);
794 bm::id64_t w64 = 0;
795 unsigned j = 0;
796 do
797 {
798 w64 |=
799 src_u->w64[j+0] | src_u->w64[j+1] | src_u->w64[j+2] | src_u->w64[j+3];
800 j += 4;
801 } while ((j < bm::set_block_digest_wave_size/2) & !w64);
802 digest &= w64 ? digest : ~(mask << wave);
803 #endif
804
805 d = bm::bmi_bslr_u64(d); // d &= d - 1;
806 } // while
807
808 BM_ASSERT(bm::calc_block_digest0(block) == digest);
809 return digest;
810 }
811
812 // --------------------------------------------------------------
813
814
815 /// Returns true if set operation is constant (bitcount)
816 inline
817 bool is_const_set_operation(set_operation op) BMNOEXCEPT
818 {
819 return (int(op) >= int(set_COUNT));
820 }
821
822 /**
823 Convert set operation to operation
824 */
825 inline
826 bm::operation setop2op(bm::set_operation op) BMNOEXCEPT
827 {
828 BM_ASSERT(op == set_AND ||
829 op == set_OR ||
830 op == set_SUB ||
831 op == set_XOR);
832 return (bm::operation) op;
833 }
834
835 //---------------------------------------------------------------------
836
837 /**
838 Structure carries pointer on bit block with all bits 1
839 @ingroup bitfunc
840 @internal
841 */
842 template<bool T> struct all_set
843 {
844 struct BM_VECT_ALIGN all_set_block
845 {
846 bm::word_t BM_VECT_ALIGN* _s[bm::set_sub_array_size] BM_VECT_ALIGN_ATTR;
847 bm::word_t BM_VECT_ALIGN _p[bm::set_block_size] BM_VECT_ALIGN_ATTR;
848 bm::word_t* _p_fullp;
849
850 all_set_block()
851 {
852 ::memset(_p, 0xFF, sizeof(_p)); // set FULL BLOCK content (all 1s)
853 if (bm::conditional<sizeof(void*) == 8>::test())
854 {
855 const unsigned long long magic_mask = 0xFFFFfffeFFFFfffe;
856 ::memcpy(&_p_fullp, &magic_mask, sizeof(magic_mask));
857 for (unsigned i = 0; i < bm::set_sub_array_size; ++i)
858 _s[i] = reinterpret_cast<bm::word_t*>(magic_mask);
859 }
860 else
861 {
862 const unsigned magic_mask = 0xFFFFfffe;
863 ::memcpy(&_p_fullp, &magic_mask, sizeof(magic_mask));
864 for (unsigned i = 0; i < bm::set_sub_array_size; ++i)
865 _s[i] = reinterpret_cast<bm::word_t*>(magic_mask);
866 }
867 }
868 };
869
870 // version with minimal branching, super-scalar friendly
871 //
872 inline
873 static bm::id64_t block_type(const bm::word_t* bp) BMNOEXCEPT
874 {
875 bm::id64_t type;
876 if (bm::conditional<sizeof(void*) == 8>::test())
877 {
878 bm::id64_t w = reinterpret_cast<unsigned long long>(bp);
879 type = (w & 3) | // FULL BLOCK or GAP
880 ((bp == _block._p) << 1);
881 type = type ? type : w;
882 }
883 else
884 {
885 unsigned w = reinterpret_cast<unsigned long>(bp);
886 type = (w & 3) | // FULL BLOCK or GAP
887 ((bp == _block._p) << 1);
888 type = type ? type : w;
889 }
890 return type;
891 }
892
893 BMFORCEINLINE
894 static bool is_full_block(const bm::word_t* bp) BMNOEXCEPT
895 { return (bp == _block._p || bp == _block._p_fullp); }
896
897 BMFORCEINLINE
898 static bool is_valid_block_addr(const bm::word_t* bp) BMNOEXCEPT
899 { return (bp && !(bp == _block._p || bp == _block._p_fullp)); }
900
901 static all_set_block _block;
902 };
903
904
905 template<bool T> typename all_set<T>::all_set_block all_set<T>::_block;
906
907 /// XOR swap two scalar variables
908 template<typename W>
909 void xor_swap(W& x, W& y) BMNOEXCEPT
910 {
911 BM_ASSERT(&x != &y);
912 x ^= y;
913 y ^= x;
914 x ^= y;
915 }
916
917 /*!
918 Fini not NULL position
919 @return index of not NULL pointer
920 @internal
921 */
922 template<typename N>
923 bool find_not_null_ptr(bm::word_t*** arr, N start, N size, N* pos) BMNOEXCEPT
924 {
925 BM_ASSERT(pos);
926 // BM_ASSERT(start < size);
927 //#if defined(BM64_AVX2) || defined(BM64_AVX512)
928 // TODO: optimization for SIMD based next ptr scan
929 #if 0
930 const unsigned unroll_factor = 4;
931 const unsigned len = (size - start);
932 const unsigned len_unr = len - (len % unroll_factor);
933 unsigned k;
934
935 arr += start;
936 for (k = 0; k < len_unr; k+=unroll_factor)
937 {
938 if (!avx2_test_all_zero_wave(arr+k))
939 {
940 if (arr[k])
941 {
942 *pos = k + start;
943 return true;
944 }
945 if (arr[k+1])
946 {
947 *pos = k + start + 1;
948 return true;
949 }
950 if (arr[k+2])
951 {
952 *pos = k + start + 2;
953 return true;
954 }
955 if (arr[k+3])
956 {
957 *pos = k + start + 3;
958 return true;
959 }
960 }
961 } // for k
962
963 for (; k < len; ++k)
964 {
965 if (arr[k])
966 {
967 *pos = k + start;
968 return true;
969 }
970 } // for k
971 #else
972 for (; start < size; ++start)
973 {
974 if (arr[start])
975 {
976 *pos = start;
977 return true;
978 }
979 } // for i
980 #endif
981 return false;
982 }
983
984
985
986
987 //---------------------------------------------------------------------
988
989 /*!
990 \brief Lexicographical comparison of two words as bit strings (reference)
991 Auxiliary implementation for testing and reference purposes.
992 \param w1 - First word.
993 \param w2 - Second word.
994 \return <0 - less, =0 - equal, >0 - greater.
995
996 @ingroup bitfunc
997 */
998 template<typename T> int wordcmp0(T w1, T w2)
999 {
1000 while (w1 != w2)
1001 {
1002 int res = (w1 & 1) - (w2 & 1);
1003 if (res != 0) return res;
1004 w1 >>= 1;
1005 w2 >>= 1;
1006 }
1007 return 0;
1008 }
1009
1010
1011 /*
1012 template<typename T> int wordcmp(T w1, T w2)
1013 {
1014 T diff = w1 ^ w2;
1015 return diff ? ((w1 & diff & (diff ^ (diff - 1)))? 1 : -1) : 0;
1016 }
1017 */
1018 /*!
1019 \brief Lexicographical comparison of two words as bit strings.
1020 Auxiliary implementation for testing and reference purposes.
1021 \param a - First word.
1022 \param b - Second word.
1023 \return <0 - less, =0 - equal, >0 - greater.
1024
1025 @ingroup bitfunc
1026 */
1027 template<typename T> int wordcmp(T a, T b)
1028 {
1029 T diff = a ^ b;
1030 return diff? ( (a & diff & -diff)? 1 : -1 ) : 0;
1031 }
1032
1033
1034 // Low bit extraction
1035 // x & (x ^ (x-1))
1036
1037
1038 // ----------------------------------------------------------------------
1039
1040
1041 /*! @brief Returns "true" if all bits in the block are 0
1042 @ingroup bitfunc
1043 */
1044 inline
1045 bool bit_is_all_zero(const bm::word_t* BMRESTRICT start) BMNOEXCEPT
1046 {
1047 #if defined(VECT_IS_ZERO_BLOCK)
1048 return VECT_IS_ZERO_BLOCK(start);
1049 #else
1050 const bm::wordop_t* BMRESTRICT blk = (bm::wordop_t*) (start);
1051 const bm::wordop_t* BMRESTRICT blk_end = (bm::wordop_t*)(start + bm::set_block_size);
1052 do
1053 {
1054 if (blk[0] | blk[1] | blk[2] | blk[3])
1055 return false;
1056 blk += 4;
1057 } while (blk < blk_end);
1058 return true;
1059 #endif
1060 }
1061
1062 // ----------------------------------------------------------------------
1063
1064 /*!
1065 \brief Checks if GAP block is all-zero.
1066 \param buf - GAP buffer pointer.
1067 \returns true if all-zero.
1068
1069 @ingroup gapfunc
1070 */
1071 BMFORCEINLINE
1072 bool gap_is_all_zero(const bm::gap_word_t* BMRESTRICT buf) BMNOEXCEPT
1073 {
1074 // (almost) branchless variant:
1075 return (!(*buf & 1u)) & (!(bm::gap_max_bits - 1 - buf[1]));
1076 }
1077
1078 /*!
1079 \brief Checks if GAP block is all-one.
1080 \param buf - GAP buffer pointer.
1081 \returns true if all-one.
1082 @ingroup gapfunc
1083 */
1084 BMFORCEINLINE
1085 bool gap_is_all_one(const bm::gap_word_t* BMRESTRICT buf) BMNOEXCEPT
1086 {
1087 return ((*buf & 1u) && (buf[1] == bm::gap_max_bits - 1));
1088 }
1089
1090 /*!
1091 \brief Returs GAP block length.
1092 \param buf - GAP buffer pointer.
1093 \returns GAP block length.
1094
1095 @ingroup gapfunc
1096 */
1097 BMFORCEINLINE
1098 bm::gap_word_t gap_length(const bm::gap_word_t* BMRESTRICT buf) BMNOEXCEPT
1099 {
1100 return (bm::gap_word_t)((*buf >> 3) + 1);
1101 }
1102
1103
1104 /*!
1105 \brief Returs GAP block capacity
1106 \param buf - GAP buffer pointer
1107 \param glevel_len - pointer on GAP header word
1108 \returns GAP block capacity.
1109
1110 @ingroup gapfunc
1111 */
1112 template<typename T>
1113 unsigned
1114 gap_capacity(const T* BMRESTRICT buf, const T* BMRESTRICT glevel_len) BMNOEXCEPT
1115 {
1116 return glevel_len[(*buf >> 1) & 3];
1117 }
1118
1119
1120 /*!
1121 \brief Returs GAP block capacity limit.
1122 \param buf - GAP buffer pointer.
1123 \param glevel_len - GAP lengths table (gap_len_table)
1124 \returns GAP block limit.
1125
1126 @ingroup gapfunc
1127 */
1128 template<typename T>
1129 unsigned
1130 gap_limit(const T* BMRESTRICT buf, const T* BMRESTRICT glevel_len) BMNOEXCEPT
1131 {
1132 return glevel_len[(*buf >> 1) & 3]-4;
1133 }
1134
1135
1136 /*!
1137 \brief Returs GAP blocks capacity level.
1138 \param buf - GAP buffer pointer.
1139 \returns GAP block capacity level.
1140
1141 @ingroup gapfunc
1142 */
1143 template<typename T>
1144 T gap_level(const T* BMRESTRICT buf) BMNOEXCEPT
1145 {
1146 return T((*buf >> 1) & 3u);
1147 }
1148
1149
1150 /*!
1151 \brief GAP block find the last set bit
1152
1153 \param buf - GAP buffer pointer.
1154 \param last - index of the last 1 bit
1155
1156 \return 0 if 1 bit was NOT found
1157
1158 @ingroup gapfunc
1159 */
1160 template<typename T>
1161 unsigned
1162 gap_find_last(const T* BMRESTRICT buf, unsigned* BMRESTRICT last) BMNOEXCEPT
1163 {
1164 BM_ASSERT(last);
1165
1166 T is_set = (*buf) & 1u;
1167 T end = T((*buf) >> 3u);
1168
1169 BM_ASSERT(buf[end] == bm::gap_max_bits - 1);
1170
1171 is_set ^= T((end-1) & 1u);
1172 if (is_set)
1173 {
1174 *last = buf[end];
1175 return is_set;
1176 }
1177 *last = buf[--end];
1178 return end;
1179 }
1180
1181 /*!
1182 \brief GAP block find the first set bit
1183
1184 \param buf - GAP buffer pointer.
1185 \param first - index of the first 1 bit
1186
1187 \return 0 if 1 bit was NOT found
1188
1189 @ingroup gapfunc
1190 */
1191 template<typename T>
1192 unsigned
1193 gap_find_first(const T* BMRESTRICT buf, unsigned* BMRESTRICT first) BMNOEXCEPT
1194 {
1195 BM_ASSERT(first);
1196
1197 T is_set = (*buf) & 1u;
1198 if (is_set)
1199 {
1200 *first = 0;
1201 return is_set;
1202 }
1203 if (buf[1] == bm::gap_max_bits - 1)
1204 return 0;
1205 *first = buf[1] + 1;
1206 return 1;
1207 }
1208
1209
1210
1211 /*
1212 \brief Binary search for the block where bit = pos located.
1213 \param buf - GAP buffer pointer.
1214 \param pos - index of the element.
1215 \param is_set - output. GAP value (0 or 1).
1216 \return GAP index.
1217 @ingroup gapfunc
1218 */
1219 template<typename T>
1220 unsigned gap_bfind(const T* BMRESTRICT buf,
1221 unsigned pos, unsigned* BMRESTRICT is_set) BMNOEXCEPT
1222 {
1223 BM_ASSERT(pos < bm::gap_max_bits);
1224 #undef VECT_GAP_BFIND // TODO: VECTOR bfind causes performance degradation
1225 #ifdef VECT_GAP_BFIND
1226 return VECT_GAP_BFIND(buf, pos, is_set);
1227 #else
1228 *is_set = (*buf) & 1;
1229
1230 unsigned start = 1;
1231 unsigned end = 1 + ((*buf) >> 3);
1232
1233 while ( start != end )
1234 {
1235 unsigned curr = (start + end) >> 1;
1236 if ( buf[curr] < pos )
1237 start = curr + 1;
1238 else
1239 end = curr;
1240 }
1241 *is_set ^= ((start-1) & 1);
1242 return start;
1243 #endif
1244 }
1245
1246
1247 /*!
1248 \brief Tests if bit = pos is true.
1249 \param buf - GAP buffer pointer.
1250 \param pos - index of the element.
1251 \return true if position is in "1" gap
1252 @ingroup gapfunc
1253 */
1254 template<typename T>
1255 unsigned gap_test(const T* BMRESTRICT buf, unsigned pos) BMNOEXCEPT
1256 {
1257 BM_ASSERT(pos < bm::gap_max_bits);
1258
1259 unsigned start = 1;
1260 unsigned end = 1 + ((*buf) >> 3);
1261 if (end - start < 10)
1262 {
1263 unsigned sv = *buf & 1;
1264 unsigned sv1= sv ^ 1;
1265 if (buf[1] >= pos) return sv;
1266 if (buf[2] >= pos) return sv1;
1267 if (buf[3] >= pos) return sv;
1268 if (buf[4] >= pos) return sv1;
1269 if (buf[5] >= pos) return sv;
1270 if (buf[6] >= pos) return sv1;
1271 if (buf[7] >= pos) return sv;
1272 if (buf[8] >= pos) return sv1;
1273 if (buf[9] >= pos) return sv;
1274 BM_ASSERT(0);
1275 }
1276 else
1277 {
1278 while (start != end)
1279 {
1280 unsigned curr = (start + end) >> 1;
1281 if (buf[curr] < pos)
1282 start = curr + 1;
1283 else
1284 end = curr;
1285 }
1286 }
1287 return ((*buf) & 1) ^ ((--start) & 1);
1288 }
1289
1290 /*!
1291 \brief Tests if bit = pos is true. Analog of bm::gap_test with SIMD unrolling.
1292 \param buf - GAP buffer pointer.
1293 \param pos - index of the element.
1294 \return true if position is in "1" gap
1295 @ingroup gapfunc
1296 */
1297 template<typename T>
1298 unsigned gap_test_unr(const T* BMRESTRICT buf, const unsigned pos) BMNOEXCEPT
1299 {
1300 BM_ASSERT(pos < bm::gap_max_bits);
1301
1302 if (pos == 0) // quick answer possible
1303 {
1304 return (*buf) & 1;
1305 }
1306 #if defined(BMSSE2OPT)
1307 unsigned res = bm::sse2_gap_test(buf, pos);
1308 BM_ASSERT(res == bm::gap_test(buf, pos));
1309 #elif defined(BMSSE42OPT)
1310 unsigned res = bm::sse42_gap_test(buf, pos);
1311 BM_ASSERT(res == bm::gap_test(buf, pos));
1312 #elif defined(BMAVX2OPT)
1313 unsigned res = bm::avx2_gap_test(buf, pos);
1314 BM_ASSERT(res == bm::gap_test(buf, pos));
1315 #else
1316 unsigned res = bm::gap_test(buf, pos);
1317 #endif
1318 return res;
1319 }
1320
1321 /*! For each non-zero block in [from, to] executes supplied functor
1322 \internal
1323 */
1324 template<typename T, typename N, typename F>
1325 void for_each_nzblock_range(T*** root,
1326 N top_size, N nb_from, N nb_to, F& f) BMNOEXCEPT
1327 {
1328 BM_ASSERT(top_size);
1329 if (nb_from > nb_to)
1330 return;
1331 unsigned i_from = unsigned(nb_from >> bm::set_array_shift);
1332 unsigned j_from = unsigned(nb_from & bm::set_array_mask);
1333 unsigned i_to = unsigned(nb_to >> bm::set_array_shift);
1334 unsigned j_to = unsigned(nb_to & bm::set_array_mask);
1335
1336 if (i_from >= top_size)
1337 return;
1338 if (i_to >= top_size)
1339 {
1340 i_to = unsigned(top_size-1);
1341 j_to = bm::set_sub_array_size-1;
1342 }
1343
1344 for (unsigned i = i_from; i <= i_to; ++i)
1345 {
1346 T** blk_blk = root[i];
1347 if (!blk_blk)
1348 continue;
1349 if ((bm::word_t*)blk_blk == FULL_BLOCK_FAKE_ADDR)
1350 {
1351 unsigned j = (i == i_from) ? j_from : 0;
1352 if (!j && (i != i_to)) // full sub-block
1353 f.add_full(bm::set_sub_total_bits);
1354 else
1355 {
1356 do
1357 {
1358 f.add_full(bm::gap_max_bits);
1359 if ((i == i_to) && (j == j_to))
1360 return;
1361 } while (++j < bm::set_sub_array_size);
1362 }
1363 }
1364 else
1365 {
1366 unsigned j = (i == i_from) ? j_from : 0;
1367 do
1368 {
1369 if (blk_blk[j])
1370 f(blk_blk[j]);
1371 if ((i == i_to) && (j == j_to))
1372 return;
1373 } while (++j < bm::set_sub_array_size);
1374 }
1375 } // for i
1376 }
1377
1378 /*! For each non-zero block executes supplied function.
1379 \internal
1380 */
1381 template<class T, class F>
1382 void for_each_nzblock(T*** root, unsigned size1, F& f)
1383 {
1384 typedef typename F::size_type size_type;
1385 for (unsigned i = 0; i < size1; ++i)
1386 {
1387 T** blk_blk = root[i];
1388 if (!blk_blk)
1389 {
1390 f.on_empty_top(i);
1391 continue;
1392 }
1393 f.on_non_empty_top(i);
1394 if ((bm::word_t*)blk_blk == FULL_BLOCK_FAKE_ADDR)
1395 {
1396 size_type r = i * bm::set_sub_array_size;
1397 unsigned j = 0;
1398 do
1399 {
1400 f(FULL_BLOCK_FAKE_ADDR, r + j);
1401 } while (++j < bm::set_sub_array_size);
1402 continue;
1403 }
1404
1405 unsigned non_empty_top = 0;
1406 size_type r = i * bm::set_sub_array_size;
1407 unsigned j = 0;
1408 do
1409 {
1410 #if defined(BM64_AVX2) || defined(BM64_AVX512)
1411 if (!avx2_test_all_zero_wave(blk_blk + j))
1412 {
1413 non_empty_top = 1;
1414 T* blk0 = blk_blk[j + 0];
1415 T* blk1 = blk_blk[j + 1];
1416 T* blk2 = blk_blk[j + 2];
1417 T* blk3 = blk_blk[j + 3];
1418
1419 size_type block_idx = r + j + 0;
1420 if (blk0)
1421 f(blk0, block_idx);
1422 else
1423 f.on_empty_block(block_idx);
1424
1425 if (blk1)
1426 f(blk1, block_idx + 1);
1427 else
1428 f.on_empty_block(block_idx + 1);
1429
1430 if (blk2)
1431 f(blk2, block_idx + 2);
1432 else
1433 f.on_empty_block(block_idx + 2);
1434
1435 if (blk3)
1436 f(blk3, block_idx + 3);
1437 else
1438 f.on_empty_block(block_idx + 3);
1439 }
1440 else
1441 {
1442 f.on_empty_block(r + j + 0); f.on_empty_block(r + j + 1);
1443 f.on_empty_block(r + j + 2); f.on_empty_block(r + j + 3);
1444 }
1445 j += 4;
1446 #elif defined(BM64_SSE4)
1447 if (!sse42_test_all_zero_wave((blk_blk + j)))
1448 {
1449 non_empty_top = 1;
1450 T* blk0 = blk_blk[j + 0];
1451 T* blk1 = blk_blk[j + 1];
1452
1453 size_type block_idx = r + j + 0;
1454 if (blk0)
1455 f(blk0, block_idx);
1456 else
1457 f.on_empty_block(block_idx);
1458
1459 ++block_idx;
1460 if (blk1)
1461 f(blk1, block_idx);
1462 else
1463 f.on_empty_block(block_idx);
1464 }
1465 else
1466 {
1467 f.on_empty_block(r + j + 0);
1468 f.on_empty_block(r + j + 1);
1469 }
1470 j += 2;
1471 #else
1472 if (blk_blk[j])
1473 {
1474 f(blk_blk[j], r + j);
1475 non_empty_top = 1;
1476 }
1477 else
1478 f.on_empty_block(r + j);
1479 ++j;
1480 #endif
1481 } while (j < bm::set_sub_array_size);
1482
1483 if (non_empty_top == 0)
1484 f.on_empty_top(i);
1485 } // for i
1486 }
1487
1488 /*! For each non-zero block executes supplied function.
1489 */
1490 template<class T, class F>
1491 void for_each_nzblock2(T*** root, unsigned size1, F& f)
1492 {
1493 #ifdef BM64_SSE4
1494 for (unsigned i = 0; i < size1; ++i)
1495 {
1496 T** blk_blk;
1497 if ((blk_blk = root[i])!=0)
1498 {
1499 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1500 {
1501 for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1502 {
1503 f(FULL_BLOCK_FAKE_ADDR);
1504 }
1505 continue;
1506 }
1507 {
1508 __m128i w0;
1509 for (unsigned j = 0; j < bm::set_sub_array_size; j+=4)
1510 {
1511 w0 = _mm_loadu_si128((__m128i*)(blk_blk + j));
1512 if (!_mm_testz_si128(w0, w0))
1513 {
1514 T* blk0 = blk_blk[j + 0];
1515 T* blk1 = blk_blk[j + 1];
1516
1517 if (blk0)
1518 f(blk0);
1519 if (blk1)
1520 f(blk1);
1521 }
1522 w0 = _mm_loadu_si128((__m128i*)(blk_blk + j + 2));
1523 if (!_mm_testz_si128(w0, w0))
1524 {
1525 T* blk0 = blk_blk[j + 2];
1526 T* blk1 = blk_blk[j + 3];
1527 if (blk0)
1528 f(blk0);
1529 if (blk1)
1530 f(blk1);
1531 }
1532 } // for j
1533 }
1534 }
1535 } // for i
1536 #elif defined(BM64_AVX2) || defined(BM64_AVX512)
1537 for (unsigned i = 0; i < size1; ++i)
1538 {
1539 T** blk_blk;
1540 if ((blk_blk = root[i]) != 0)
1541 {
1542 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1543 {
1544 for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1545 {
1546 f(FULL_BLOCK_FAKE_ADDR);
1547 }
1548 continue;
1549 }
1550 {
1551 for (unsigned j = 0; j < bm::set_sub_array_size; j += 4)
1552 {
1553 __m256i w0 = _mm256_loadu_si256((__m256i*)(blk_blk + j));
1554 if (!_mm256_testz_si256(w0, w0))
1555 {
1556 // as a variant could use: blk0 = (T*)_mm256_extract_epi64(w0, 0);
1557 // but it measures marginally slower
1558 T* blk0 = blk_blk[j + 0];
1559 T* blk1 = blk_blk[j + 1];
1560 T* blk2 = blk_blk[j + 2];
1561 T* blk3 = blk_blk[j + 3];
1562 if (blk0)
1563 f(blk0);
1564 if (blk1)
1565 f(blk1);
1566 if (blk2)
1567 f(blk2);
1568 if (blk3)
1569 f(blk3);
1570 }
1571 } // for j
1572 }
1573 }
1574 } // for i
1575
1576 #else // non-SIMD mode
1577 for (unsigned i = 0; i < size1; ++i)
1578 {
1579 T** blk_blk;
1580 if ((blk_blk = root[i])!=0)
1581 {
1582 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1583 {
1584 for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1585 {
1586 f(FULL_BLOCK_FAKE_ADDR);
1587 }
1588 continue;
1589 }
1590 unsigned j = 0;
1591 do
1592 {
1593 if (blk_blk[j])
1594 f(blk_blk[j]);
1595 if (blk_blk[j+1])
1596 f(blk_blk[j+1]);
1597 if (blk_blk[j+2])
1598 f(blk_blk[j+2]);
1599 if (blk_blk[j+3])
1600 f(blk_blk[j+3]);
1601 j += 4;
1602 } while (j < bm::set_sub_array_size);
1603 }
1604 } // for i
1605 #endif
1606 }
1607
1608
1609 /*! For each non-zero block executes supplied function-predicate.
1610 Function returns if function-predicate returns true
1611 */
1612 template<typename T, typename BI, typename F>
1613 bool for_each_nzblock_if(T*** root, BI size1, F& f) BMNOEXCEPT
1614 {
1615 BI block_idx = 0;
1616 for (BI i = 0; i < size1; ++i)
1617 {
1618 T** blk_blk = root[i];
1619 if (!blk_blk)
1620 {
1621 block_idx += bm::set_sub_array_size;
1622 continue;
1623 }
1624 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1625 {
1626 for (unsigned j = 0; j < bm::set_sub_array_size; ++j, ++block_idx)
1627 {
1628 if (f(FULL_BLOCK_FAKE_ADDR, block_idx))
1629 return true;
1630 } // for j
1631 continue;
1632 }
1633
1634 for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1635 {
1636 if (blk_blk[j])
1637 if (f(blk_blk[j], block_idx))
1638 return true;
1639 } // for j
1640 } // for i
1641 return false;
1642 }
1643
1644 /*! For each block executes supplied function.
1645 */
1646 template<class T, class F, typename BLOCK_IDX>
1647 void for_each_block(T*** root, unsigned size1, F& f, BLOCK_IDX start)
1648 {
1649 BLOCK_IDX block_idx = start;
1650 for (unsigned i = 0; i < size1; ++i)
1651 {
1652 T** blk_blk = root[i];
1653 if (blk_blk)
1654 {
1655 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1656 {
1657 for (unsigned j = 0; j < bm::set_sub_array_size; ++j, ++block_idx)
1658 {
1659 f(FULL_BLOCK_FAKE_ADDR, block_idx);
1660 }
1661 continue;
1662 }
1663 for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1664 {
1665 f(blk_blk[j], block_idx);
1666 }
1667 }
1668 else
1669 {
1670 for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1671 {
1672 f(0, block_idx);
1673 }
1674 }
1675 }
1676 }
1677
1678
1679
1680 /*! Special BM optimized analog of STL for_each
1681 */
1682 template<class T, class F> F bmfor_each(T first, T last, F f)
1683 {
1684 do
1685 {
1686 f(*first);
1687 ++first;
1688 } while (first < last);
1689 return f;
1690 }
1691
1692 /*! Computes SUM of all elements of the sequence
1693 */
1694 template<typename T>
1695 bm::id64_t sum_arr(const T* first, const T* last) BMNOEXCEPT
1696 {
1697 bm::id64_t sum = 0;
1698 for (;first < last; ++first)
1699 sum += *first;
1700 return sum;
1701 }
1702
1703 /*!
1704 Extract short (len=1) exceptions from the GAP block
1705 \param buf - GAP buffer to split
1706 \param arr0 - [OUT] list of isolates 0 positions (clear list)
1707 \param arr1 - [OUT] list of isolated 1 positions (set list)
1708 \param arr0_cnt - [OUT] arr0 size
1709 \param arr1_cnt -
1710 @ingroup gapfunc
1711 */
1712 template<typename T>
1713 void gap_split(const T* buf,
1714 T* arr0, T* arr1, T& arr0_cnt, T& arr1_cnt) BMNOEXCEPT
1715 {
1716 const T* pcurr = buf;
1717 unsigned len = (*pcurr >> 3);
1718 const T* pend = pcurr + len;
1719
1720 T cnt0, cnt1;
1721 cnt0 = cnt1 = 0;
1722 unsigned is_set = (*buf & 1);
1723
1724 if (*pcurr == 0)
1725 {
1726 if (is_set)
1727 {
1728 arr1[cnt1] = *pcurr;
1729 ++cnt1;
1730 }
1731 else
1732 {
1733 arr0[cnt0] = *pcurr;
1734 ++cnt0;
1735 }
1736 }
1737 T prev = *pcurr;
1738 ++pcurr;
1739
1740 while (pcurr <= pend)
1741 {
1742 is_set ^= 1;
1743 T delta = *pcurr - prev;
1744 if (delta == 1)
1745 {
1746 if (is_set)
1747 {
1748 arr1[cnt1] = prev;
1749 ++cnt1;
1750 }
1751 else
1752 {
1753 arr0[cnt0] = prev;
1754 ++cnt0;
1755 }
1756 }
1757 prev = *pcurr++;
1758 } // while
1759
1760 arr0_cnt = cnt0;
1761 arr1_cnt = cnt1;
1762 }
1763
1764
1765 /*!
1766 \brief Calculates number of bits ON in GAP buffer.
1767 \param buf - GAP buffer pointer.
1768 \param dsize - buffer size
1769 \return Number of non-zero bits.
1770 @ingroup gapfunc
1771 */
1772 template<typename T>
1773 unsigned gap_bit_count(const T* buf, unsigned dsize=0) BMNOEXCEPT
1774 {
1775 const T* pcurr = buf;
1776 if (dsize == 0)
1777 dsize = (*pcurr >> 3);
1778
1779 const T* pend = pcurr + dsize;
1780
1781 unsigned bits_counter = 0;
1782 ++pcurr;
1783
1784 if (*buf & 1)
1785 {
1786 bits_counter += *pcurr + 1;
1787 ++pcurr;
1788 }
1789 for (++pcurr; pcurr <= pend; pcurr += 2)
1790 bits_counter += *pcurr - *(pcurr-1);
1791 return bits_counter;
1792 }
1793
1794 /*!
1795 \brief Calculates number of bits ON in GAP buffer. Loop unrolled version.
1796 \param buf - GAP buffer pointer.
1797 \return Number of non-zero bits.
1798 @ingroup gapfunc
1799 */
1800 template<typename T>
1801 unsigned gap_bit_count_unr(const T* buf) BMNOEXCEPT
1802 {
1803 const T* pcurr = buf;
1804 unsigned dsize = (*pcurr >> 3);
1805
1806 unsigned cnt = 0;
1807 pcurr = buf + 1; // set up start position
1808 T first_one = *buf & 1;
1809 if (first_one)
1810 {
1811 cnt += *pcurr + 1;
1812 ++pcurr;
1813 }
1814 ++pcurr; // set GAP to 1
1815
1816 #if defined(BMAVX2OPT) || defined(BMAVX512OPT)
1817 if (dsize > 34)
1818 {
1819 const unsigned unr_factor = 32;
1820 unsigned waves = (dsize-2) / unr_factor;
1821 pcurr = avx2_gap_sum_arr(pcurr, waves, &cnt);
1822 }
1823 #elif defined(BMSSE42OPT) || defined(BMSSE2OPT)
1824 if (dsize > 18)
1825 {
1826 const unsigned unr_factor = 16;
1827 unsigned waves = (dsize - 2) / unr_factor;
1828 pcurr = sse2_gap_sum_arr(pcurr, waves, &cnt);
1829 }
1830 #else
1831 if (dsize > 10)
1832 {
1833 const unsigned unr_factor = 8;
1834 unsigned waves = (dsize - 2) / unr_factor;
1835 for (unsigned i = 0; i < waves; i += unr_factor)
1836 {
1837 cnt += pcurr[0] - pcurr[0 - 1];
1838 cnt += pcurr[2] - pcurr[2 - 1];
1839 cnt += pcurr[4] - pcurr[4 - 1];
1840 cnt += pcurr[6] - pcurr[6 - 1];
1841
1842 pcurr += unr_factor;
1843 } // for
1844 }
1845 #endif
1846
1847 const T* pend = buf + dsize;
1848 for ( ; pcurr <= pend ; pcurr+=2)
1849 {
1850 cnt += *pcurr - *(pcurr - 1);
1851 }
1852 BM_ASSERT(cnt == bm::gap_bit_count(buf));
1853 return cnt;
1854 }
1855
1856
1857
1858 /*!
1859 \brief Counts 1 bits in GAP buffer in the closed [left, right] range.
1860 \param buf - GAP buffer pointer.
1861 \param left - leftmost bit index to start from
1862 \param right- rightmost bit index
1863 \return Number of non-zero bits.
1864 @ingroup gapfunc
1865 */
1866 template<typename T>
1867 unsigned gap_bit_count_range(const T* const buf,
1868 unsigned left, unsigned right) BMNOEXCEPT
1869 {
1870 BM_ASSERT(left <= right);
1871 BM_ASSERT(right < bm::gap_max_bits);
1872
1873 const T* pcurr = buf;
1874 const T* pend = pcurr + (*pcurr >> 3);
1875
1876 unsigned bits_counter = 0;
1877 unsigned is_set;
1878 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1879 is_set = ~(is_set - 1u); // 0xFFF.. if true (mask for branchless code)
1880
1881 pcurr = buf + start_pos;
1882 if (right <= *pcurr) // we are in the target gap right now
1883 {
1884 bits_counter = unsigned(right - left + 1u) & is_set; // & is_set == if(is_set)
1885 return bits_counter;
1886 }
1887 bits_counter += unsigned(*pcurr - left + 1u) & is_set;
1888
1889 unsigned prev_gap = *pcurr++;
1890 for (is_set ^= ~0u; right > *pcurr; is_set ^= ~0u)
1891 {
1892 bits_counter += (*pcurr - prev_gap) & is_set;
1893 if (pcurr == pend)
1894 return bits_counter;
1895 prev_gap = *pcurr++;
1896 }
1897 bits_counter += unsigned(right - prev_gap) & is_set;
1898 return bits_counter;
1899 }
1900
1901 /*!
1902 \brief Test if all bits are 1 in GAP buffer in the [left, right] range.
1903 \param buf - GAP buffer pointer.
1904 \param left - leftmost bit index to start from
1905 \param right- rightmost bit index
1906 \return true if all bits are "11111"
1907 @ingroup gapfunc
1908 */
1909 template<typename T>
1910 bool gap_is_all_one_range(const T* const BMRESTRICT buf,
1911 unsigned left, unsigned right) BMNOEXCEPT
1912 {
1913 BM_ASSERT(left <= right);
1914 BM_ASSERT(right < bm::gap_max_bits);
1915
1916 unsigned is_set;
1917 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1918 if (!is_set) // GAP is 0
1919 return false;
1920 const T* const pcurr = buf + start_pos;
1921 return (right <= *pcurr);
1922 }
1923
1924 /*!
1925 \brief Test if any bits are 1 in GAP buffer in the [left, right] range.
1926 \param buf - GAP buffer pointer.
1927 \param left - leftmost bit index to start from
1928 \param right- rightmost bit index
1929 \return true if at least 1 "00010"
1930 @ingroup gapfunc
1931 */
1932 template<typename T>
1933 bool gap_any_range(const T* const BMRESTRICT buf,
1934 unsigned left, unsigned right) BMNOEXCEPT
1935 {
1936 BM_ASSERT(left <= right);
1937 BM_ASSERT(right < bm::gap_max_bits);
1938
1939 unsigned is_set;
1940 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1941 const T* const pcurr = buf + start_pos;
1942
1943 if (!is_set) // start GAP is 0 ...
1944 {
1945 if (right <= *pcurr) // ...bit if the interval goes into at least 1 blk
1946 return false; // .. nope
1947 return true;
1948 }
1949 return true;
1950 }
1951
1952 /*!
1953 \brief Test if any bits are 1 in GAP buffer in the [left, right] range
1954 and flanked with 0s
1955 \param buf - GAP buffer pointer.
1956 \param left - leftmost bit index to start from
1957 \param right- rightmost bit index
1958 \return true if "011110"
1959 @ingroup gapfunc
1960 */
1961 template<typename T>
1962 bool gap_is_interval(const T* const BMRESTRICT buf,
1963 unsigned left, unsigned right) BMNOEXCEPT
1964 {
1965 BM_ASSERT(left <= right);
1966 BM_ASSERT(left > 0); // cannot check left-1 otherwise
1967 BM_ASSERT(right < bm::gap_max_bits-1); // cannot check right+1 otherwise
1968
1969 unsigned is_set;
1970 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1971
1972 const T* pcurr = buf + start_pos;
1973 if (!is_set || (right != *pcurr) || (start_pos <= 1))
1974 return false;
1975 --pcurr;
1976 if (*pcurr != left-1)
1977 return false;
1978 return true;
1979 }
1980
1981 /**
1982 \brief Searches for the last 1 bit in the 111 interval of a GAP block
1983 \param buf - BIT block buffer
1984 \param nbit - bit index to start checking from
1985 \param pos - [out] found value
1986
1987 \return false if not found
1988 @ingroup gapfunc
1989 */
1990 template<typename T>
1991 bool gap_find_interval_end(const T* const BMRESTRICT buf,
1992 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
1993 {
1994 BM_ASSERT(pos);
1995 BM_ASSERT(nbit < bm::gap_max_bits);
1996
1997 unsigned is_set;
1998 unsigned start_pos = bm::gap_bfind(buf, nbit, &is_set);
1999 if (!is_set)
2000 return false;
2001 *pos = buf[start_pos];
2002 return true;
2003 }
2004
2005
2006 /**
2007 \brief Searches for the first 1 bit in the 111 interval of a GAP block
2008 \param buf - BIT block buffer
2009 \param nbit - bit index to start checking from
2010 \param pos - [out] found value
2011
2012 \return false if not found
2013 @ingroup gapfunc
2014 */
2015 template<typename T>
2016 bool gap_find_interval_start(const T* const BMRESTRICT buf,
2017 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
2018 {
2019 BM_ASSERT(pos);
2020 BM_ASSERT(nbit < bm::gap_max_bits);
2021
2022 unsigned is_set;
2023 unsigned start_pos = bm::gap_bfind(buf, nbit, &is_set);
2024 if (!is_set)
2025 return false;
2026 --start_pos;
2027 if (!start_pos)
2028 *pos = 0;
2029 else
2030 *pos = buf[start_pos]+1;
2031 return true;
2032 }
2033
2034
2035 /*!
2036 \brief GAP block find position for the rank
2037
2038 \param block - bit block buffer pointer
2039 \param rank - rank to find (must be > 0)
2040 \param nbit_from - start bit position in block
2041 \param nbit_pos - found position
2042
2043 \return 0 if position with rank was found, or
2044 the remaining rank (rank - population count)
2045
2046 @ingroup gapfunc
2047 */
2048 template<typename T, typename SIZE_TYPE>
2049 SIZE_TYPE gap_find_rank(const T* const block,
2050 SIZE_TYPE rank,
2051 unsigned nbit_from,
2052 unsigned& nbit_pos) BMNOEXCEPT
2053 {
2054 BM_ASSERT(block);
2055 BM_ASSERT(rank);
2056
2057 const T* pcurr = block;
2058 const T* pend = pcurr + (*pcurr >> 3);
2059
2060 unsigned bits_counter = 0;
2061 unsigned is_set;
2062 unsigned start_pos = bm::gap_bfind(block, nbit_from, &is_set);
2063 is_set = ~(is_set - 1u); // 0xFFF.. if true (mask for branchless code)
2064
2065 pcurr = block + start_pos;
2066 bits_counter += unsigned(*pcurr - nbit_from + 1u) & is_set;
2067 if (bits_counter >= rank) // found!
2068 {
2069 nbit_pos = nbit_from + unsigned(rank) - 1u;
2070 return 0;
2071 }
2072 rank -= bits_counter;
2073 unsigned prev_gap = *pcurr++;
2074 for (is_set ^= ~0u; pcurr <= pend; is_set ^= ~0u)
2075 {
2076 bits_counter = (*pcurr - prev_gap) & is_set;
2077 if (bits_counter >= rank) // found!
2078 {
2079 nbit_pos = prev_gap + unsigned(rank);
2080 return 0;
2081 }
2082 rank -= bits_counter;
2083 prev_gap = *pcurr++;
2084 } // for
2085
2086 return rank;
2087 }
2088
2089
2090
2091 /*!
2092 \brief Counts 1 bits in GAP buffer in the closed [0, right] range.
2093 \param buf - GAP buffer pointer.
2094 \param right- rightmost bit index
2095 \param is_corrected - if true the result will be rank corrected
2096 if right bit == true count=count-1
2097 \return Number of non-zero bits
2098 @ingroup gapfunc
2099 */
2100 template<typename T>
2101 unsigned gap_bit_count_to(const T* const buf, T right,
2102 bool is_corrected=false) BMNOEXCEPT
2103 {
2104 const T* pcurr = buf;
2105 const T* pend = pcurr + (*pcurr >> 3);
2106
2107 unsigned bits_counter = 0;
2108 unsigned is_set = ~((unsigned(*buf) & 1u) - 1u); // 0xFFF.. if true (mask for branchless code)
2109 BM_ASSERT(is_set == 0u || is_set == ~0u);
2110 pcurr = buf + 1;
2111
2112 if (right <= *pcurr) // we are in the target block right now
2113 {
2114 bits_counter = (right + 1u) & is_set; // & is_set == if (is_set)
2115 bits_counter -= (is_set & is_corrected);
2116 return bits_counter;
2117 }
2118 bits_counter += (*pcurr + 1u) & is_set;
2119
2120 unsigned prev_gap = *pcurr++;
2121 for (is_set ^= ~0u; right > *pcurr; is_set ^= ~0u)
2122 {
2123 bits_counter += (*pcurr - prev_gap) & is_set;
2124 if (pcurr == pend)
2125 {
2126 bits_counter -= (is_set & is_corrected);
2127 return bits_counter;
2128 }
2129 prev_gap = *pcurr++;
2130 }
2131 bits_counter += (right - prev_gap) & is_set;
2132 bits_counter -= (is_set & is_corrected);
2133 return bits_counter;
2134 }
2135
2136
2137 /*!
2138 D-GAP block for_each algorithm
2139
2140 D-Gap Functor is called for each element but last one.
2141
2142 \param gap_buf - GAP buffer
2143 \param func - functor object
2144
2145 */
2146 template<class T, class Func>
2147 void for_each_dgap(const T* gap_buf, Func& func)
2148 {
2149 const T* pcurr = gap_buf;
2150 const T* pend = pcurr + (*pcurr >> 3);
2151 ++pcurr;
2152
2153 T prev = *pcurr;
2154 func((T)(prev + 1)); // first element incremented to avoid 0
2155 ++pcurr;
2156 do
2157 {
2158 func((T)(*pcurr - prev)); // all others are [N] - [N-1]
2159 prev = *pcurr;
2160 } while (++pcurr < pend);
2161 }
2162
2163 /** d-Gap copy functor
2164 @internal
2165 */
2166 template<typename T> struct d_copy_func
2167 {
2168 d_copy_func(T* dg_buf) : dgap_buf_(dg_buf) {}
2169 void operator()(T dgap) { *dgap_buf_++ = dgap; }
2170
2171 T* dgap_buf_;
2172 };
2173
2174 /*!
2175 \brief Convert GAP buffer into D-GAP buffer
2176
2177 Delta GAP representation is DGAP[N] = GAP[N] - GAP[N-1]
2178
2179 \param gap_buf - GAP buffer
2180 \param dgap_buf - Delta-GAP buffer
2181 \param copy_head - flag to copy GAP header
2182
2183 \internal
2184
2185 @ingroup gapfunc
2186 */
2187 template<typename T>
2188 T* gap_2_dgap(const T* BMRESTRICT gap_buf,
2189 T* BMRESTRICT dgap_buf, bool copy_head=true) BMNOEXCEPT
2190 {
2191 if (copy_head) // copy GAP header
2192 {
2193 *dgap_buf++ = *gap_buf;
2194 }
2195
2196 d_copy_func<T> copy_func(dgap_buf);
2197 for_each_dgap<T, d_copy_func<T> >(gap_buf, copy_func);
2198 return copy_func.dgap_buf_;
2199 }
2200
2201 /*!
2202 \brief Convert D-GAP buffer into GAP buffer
2203
2204 GAP representation is GAP[N] = DGAP[N] + DGAP[N-1]
2205
2206 \param dgap_buf - Delta-GAP buffer
2207 \param gap_header - GAP header word
2208 \param gap_buf - GAP buffer
2209
2210 \internal
2211 @ingroup gapfunc
2212 */
2213 template<typename T>
2214 void dgap_2_gap(const T* BMRESTRICT dgap_buf,
2215 T* BMRESTRICT gap_buf, T gap_header=0) BMNOEXCEPT
2216 {
2217 const T* pcurr = dgap_buf;
2218 unsigned len;
2219 if (!gap_header) // GAP header is already part of the stream
2220 {
2221 len = *pcurr >> 3;
2222 *gap_buf++ = *pcurr++; // copy GAP header
2223 }
2224 else // GAP header passed as a parameter
2225 {
2226 len = gap_header >> 3;
2227 *gap_buf++ = gap_header; // assign GAP header
2228 }
2229 --len; // last element is actually not encoded
2230 const T* pend = pcurr + len;
2231
2232 *gap_buf = *pcurr++; // copy first element
2233 if (*gap_buf == 0)
2234 *gap_buf = 65535; // fix +1 overflow
2235 else
2236 *gap_buf = T(*gap_buf - 1);
2237
2238 for (++gap_buf; pcurr < pend; ++pcurr)
2239 {
2240 T prev = *(gap_buf-1); // don't remove temp(undef expression!)
2241 *gap_buf++ = T(*pcurr + prev);
2242 }
2243 *gap_buf = 65535; // add missing last element
2244 }
2245
2246
2247 /*!
2248 \brief Lexicographical comparison of GAP buffers.
2249 \param buf1 - First GAP buffer pointer.
2250 \param buf2 - Second GAP buffer pointer.
2251 \return <0 - less, =0 - equal, >0 - greater.
2252
2253 @ingroup gapfunc
2254 */
2255 template<typename T>
2256 int gapcmp(const T* buf1, const T* buf2) BMNOEXCEPT
2257 {
2258 const T* pcurr1 = buf1;
2259 const T* pend1 = pcurr1 + (*pcurr1 >> 3);
2260 unsigned bitval1 = *buf1 & 1;
2261 ++pcurr1;
2262
2263 const T* pcurr2 = buf2;
2264 unsigned bitval2 = *buf2 & 1;
2265 ++pcurr2;
2266
2267 while (pcurr1 <= pend1)
2268 {
2269 if (*pcurr1 == *pcurr2)
2270 {
2271 if (bitval1 != bitval2)
2272 {
2273 return (bitval1) ? 1 : -1;
2274 }
2275 }
2276 else
2277 {
2278 if (bitval1 == bitval2)
2279 {
2280 if (bitval1)
2281 {
2282 return (*pcurr1 < *pcurr2) ? -1 : 1;
2283 }
2284 else
2285 {
2286 return (*pcurr1 < *pcurr2) ? 1 : -1;
2287 }
2288 }
2289 else
2290 {
2291 return (bitval1) ? 1 : -1;
2292 }
2293 }
2294
2295 ++pcurr1; ++pcurr2;
2296
2297 bitval1 ^= 1;
2298 bitval2 ^= 1;
2299 }
2300
2301 return 0;
2302 }
2303
2304 /*!
2305 \brief Find first bit which is different between two GAP-blocks
2306 \param buf1 - block 1
2307 \param buf2 - block 2
2308 \param pos - out - position of difference (undefined if blocks are equal)
2309 \return true if difference was found
2310
2311 @ingroup gapfunc
2312 */
2313 template<typename T>
2314 bool gap_find_first_diff(const T* BMRESTRICT buf1,
2315 const T* BMRESTRICT buf2,
2316 unsigned* BMRESTRICT pos) BMNOEXCEPT
2317 {
2318 BM_ASSERT(buf1 && buf2 && pos);
2319
2320 const T* pcurr1 = buf1;
2321 const T* pend1 = pcurr1 + (*pcurr1 >> 3);
2322 const T* pcurr2 = buf2;
2323 for (++pcurr1, ++pcurr2; pcurr1 <= pend1; ++pcurr1, ++pcurr2)
2324 {
2325 if (*pcurr1 != *pcurr2)
2326 {
2327 *pos = 1 + ((*pcurr1 < *pcurr2) ? *pcurr1 : *pcurr2);
2328 return true;
2329 }
2330 } // for
2331 return false;
2332 }
2333
2334 // -------------------------------------------------------------------------
2335 //
2336
2337 /*!
2338 \brief Abstract operation for GAP buffers.
2339 Receives functor F as a template argument
2340 \param dest - destination memory buffer.
2341 \param vect1 - operand 1 GAP encoded buffer.
2342 \param vect1_mask - XOR mask for starting bitflag for vector1
2343 can be 0 or 1 (1 inverts the vector)
2344 \param vect2 - operand 2 GAP encoded buffer.
2345 \param vect2_mask - same as vect1_mask
2346 \param f - operation functor.
2347 \param dlen - destination length after the operation
2348
2349 \note Internal function.
2350 @internal
2351
2352 @ingroup gapfunc
2353 */
2354 template<typename T, class F>
2355 void gap_buff_op(T* BMRESTRICT dest,
2356 const T* BMRESTRICT vect1,
2357 unsigned vect1_mask,
2358 const T* BMRESTRICT vect2,
2359 unsigned vect2_mask,
2360 F& f,
2361 unsigned& dlen) BMNOEXCEPT2
2362 {
2363 const T* cur1 = vect1;
2364 const T* cur2 = vect2;
2365
2366 T bitval1 = (T)((*cur1++ & 1) ^ vect1_mask);
2367 T bitval2 = (T)((*cur2++ & 1) ^ vect2_mask);
2368
2369 T bitval = (T) f(bitval1, bitval2);
2370 T bitval_prev = bitval;
2371
2372 T* res = dest;
2373 *res = bitval;
2374 ++res;
2375
2376 T c1 = *cur1; T c2 = *cur2;
2377 while (1)
2378 {
2379 bitval = (T) f(bitval1, bitval2);
2380
2381 // Check if GAP value changes and we need to
2382 // start the next one
2383 //
2384 res += (bitval != bitval_prev);
2385 bitval_prev = bitval;
2386 if (c1 < c2) // (*cur1 < *cur2)
2387 {
2388 *res = c1;
2389 ++cur1; c1 = *cur1;
2390 bitval1 ^= 1;
2391 }
2392 else // >=
2393 {
2394 *res = c2;
2395 if (c2 < c1) // (*cur2 < *cur1)
2396 {
2397 bitval2 ^= 1;
2398 }
2399 else // equal
2400 {
2401 if (c2 == (bm::gap_max_bits - 1))
2402 break;
2403
2404 ++cur1; c1 = *cur1;
2405 bitval1 ^= 1; bitval2 ^= 1;
2406 }
2407 ++cur2; c2 = *cur2;
2408 }
2409 } // while
2410
2411 dlen = (unsigned)(res - dest);
2412 *dest = (T)((*dest & 7) + (dlen << 3));
2413 }
2414
2415
2416 /*!
2417 \brief Abstract operation for GAP buffers (predicts legth)
2418 Receives functor F as a template argument
2419 \param vect1 - operand 1 GAP encoded buffer.
2420 \param vect2 - operand 2 GAP encoded buffer.
2421 \param f - operation functor.
2422 \param dlen - destination length after the operation
2423 \param limit - maximum target length limit,
2424 returns false if limit is reached
2425 \return true if operation would be successfull or
2426 false if limit reached
2427
2428 \note Internal function.
2429 @internal
2430
2431 @ingroup gapfunc
2432 */
2433 template<typename T, class F>
2434 bool gap_buff_dry_op(const T* BMRESTRICT vect1,
2435 const T* BMRESTRICT vect2,
2436 F& f,
2437 unsigned& dlen,
2438 unsigned limit) BMNOEXCEPT2
2439 {
2440 const T* cur1 = vect1;
2441 const T* cur2 = vect2;
2442
2443 T bitval1 = (T)((*cur1++ & 1));
2444 T bitval2 = (T)((*cur2++ & 1));
2445
2446 T bitval = (T) f(bitval1, bitval2);
2447 T bitval_prev = bitval;
2448
2449 unsigned len = 1;
2450
2451 T c1 = *cur1; T c2 = *cur2;
2452 while (1)
2453 {
2454 bitval = (T) f(bitval1, bitval2);
2455
2456 // Check if GAP value changes and we need to
2457 // start the next one
2458 //
2459 len += (bitval != bitval_prev);
2460 if (len > limit)
2461 return false;
2462 bitval_prev = bitval;
2463 if (c1 < c2)
2464 {
2465 ++cur1; c1 = *cur1;
2466 bitval1 ^= 1;
2467 }
2468 else // >=
2469 {
2470 if (c2 < c1) // (*cur2 < *cur1)
2471 {
2472 bitval2 ^= 1;
2473 }
2474 else // equal
2475 {
2476 if (c2 == (bm::gap_max_bits - 1))
2477 break;
2478
2479 ++cur1; c1 = *cur1;
2480 bitval1 ^= 1; bitval2 ^= 1;
2481 }
2482 ++cur2; c2 = *cur2;
2483 }
2484
2485 } // while
2486
2487 dlen = len;
2488 return true;
2489 }
2490
2491
2492 /*!
2493 \brief Abstract distance test operation for GAP buffers.
2494 Receives functor F as a template argument
2495 \param vect1 - operand 1 GAP encoded buffer.
2496 \param vect1_mask - XOR mask for starting bitflag for vector1
2497 can be 0 or 1 (1 inverts the vector)
2498 \param vect2 - operand 2 GAP encoded buffer.
2499 \param vect2_mask - same as vect1_mask
2500 \param f - operation functor.
2501 \note Internal function.
2502 \return non zero value if operation result returns any 1 bit
2503
2504 @ingroup gapfunc
2505 */
2506 template<typename T, class F>
2507 unsigned gap_buff_any_op(const T* BMRESTRICT vect1,
2508 unsigned vect1_mask,
2509 const T* BMRESTRICT vect2,
2510 unsigned vect2_mask,
2511 F f) BMNOEXCEPT2
2512 {
2513 const T* cur1 = vect1;
2514 const T* cur2 = vect2;
2515
2516 unsigned bitval1 = (*cur1++ & 1) ^ vect1_mask;
2517 unsigned bitval2 = (*cur2++ & 1) ^ vect2_mask;
2518
2519 unsigned bitval = f(bitval1, bitval2);
2520 if (bitval)
2521 return bitval;
2522 unsigned bitval_prev = bitval;
2523
2524 while (1)
2525 {
2526 bitval = f(bitval1, bitval2);
2527 if (bitval)
2528 return bitval;
2529
2530 if (bitval != bitval_prev)
2531 bitval_prev = bitval;
2532
2533 if (*cur1 < *cur2)
2534 {
2535 ++cur1;
2536 bitval1 ^= 1;
2537 }
2538 else // >=
2539 {
2540 if (*cur2 < *cur1)
2541 {
2542 bitval2 ^= 1;
2543 }
2544 else // equal
2545 {
2546 if (*cur2 == (bm::gap_max_bits - 1))
2547 {
2548 break;
2549 }
2550
2551 ++cur1;
2552 bitval1 ^= 1;
2553 bitval2 ^= 1;
2554 }
2555 ++cur2;
2556 }
2557
2558 } // while
2559
2560 return 0;
2561 }
2562
2563
2564
2565 /*!
2566 \brief Abstract distance(similarity) operation for GAP buffers.
2567 Receives functor F as a template argument
2568 \param vect1 - operand 1 GAP encoded buffer.
2569 \param vect2 - operand 2 GAP encoded buffer.
2570 \param f - operation functor.
2571 \note Internal function.
2572
2573 @ingroup gapfunc
2574 */
2575 template<typename T, class F>
2576 unsigned gap_buff_count_op(const T* vect1, const T* vect2, F f) BMNOEXCEPT2
2577 {
2578 unsigned count;// = 0;
2579 const T* cur1 = vect1;
2580 const T* cur2 = vect2;
2581
2582 unsigned bitval1 = (*cur1++ & 1);
2583 unsigned bitval2 = (*cur2++ & 1);
2584 unsigned bitval = count = f(bitval1, bitval2);
2585 unsigned bitval_prev = bitval;
2586
2587 //if (bitval) ++count;
2588
2589 T res, res_prev;
2590 res = res_prev = 0;
2591
2592 while (1)
2593 {
2594 bitval = f(bitval1, bitval2);
2595
2596 // Check if GAP value changes and we need to
2597 // start the next one.
2598 if (bitval != bitval_prev)
2599 {
2600 bitval_prev = bitval;
2601 res_prev = res;
2602 }
2603
2604 if (*cur1 < *cur2)
2605 {
2606 res = *cur1;
2607 if (bitval)
2608 {
2609 count += res - res_prev;
2610 res_prev = res;
2611 }
2612 ++cur1; bitval1 ^= 1;
2613 }
2614 else // >=
2615 {
2616 res = *cur2;
2617 if (bitval)
2618 {
2619 count += res - res_prev;
2620 res_prev = res;
2621 }
2622 if (*cur2 < *cur1)
2623 {
2624 bitval2 ^= 1;
2625 }
2626 else // equal
2627 {
2628 if (*cur2 == (bm::gap_max_bits - 1))
2629 break;
2630
2631 ++cur1;
2632 bitval1 ^= 1; bitval2 ^= 1;
2633 }
2634 ++cur2;
2635 }
2636
2637 } // while
2638
2639 return count;
2640 }
2641
2642
2643 #ifdef __GNUG__
2644 #pragma GCC diagnostic push
2645 #pragma GCC diagnostic ignored "-Wconversion"
2646 #endif
2647
2648 /*!
2649 \brief Sets or clears bit in the GAP buffer.
2650
2651 \param val - new bit value
2652 \param buf - GAP buffer.
2653 \param pos - Index of bit to set.
2654 \param is_set - (OUT) flag if bit was actually set.
2655
2656 \return New GAP buffer length.
2657
2658 @ingroup gapfunc
2659 */
2660 template<typename T>
2661 unsigned gap_set_value(unsigned val,
2662 T* BMRESTRICT buf,
2663 unsigned pos,
2664 unsigned* BMRESTRICT is_set) BMNOEXCEPT
2665 {
2666 BM_ASSERT(pos < bm::gap_max_bits);
2667
2668 unsigned curr = bm::gap_bfind(buf, pos, is_set);
2669 T end = (T)(*buf >> 3);
2670 if (*is_set == val)
2671 {
2672 *is_set = 0;
2673 return end;
2674 }
2675 *is_set = 1;
2676
2677 T* pcurr = buf + curr;
2678 T* pprev = pcurr - 1;
2679 T* pend = buf + end;
2680
2681 // Special case, first bit GAP operation. There is no platform beside it.
2682 // initial flag must be inverted.
2683 if (!pos)
2684 {
2685 *buf ^= 1;
2686 if (buf[1]) // We need to insert a 1 bit GAP here
2687 {
2688 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2689 buf[1] = 0;
2690 ++end;
2691 }
2692 else // Only 1 bit in the GAP. We need to delete the first GAP.
2693 {
2694 pprev = buf + 1; pcurr = pprev + 1;
2695 goto copy_gaps;
2696 }
2697 }
2698 else
2699 if (curr > 1 && ((unsigned)(*pprev))+1 == pos) // Left border bit
2700 {
2701 ++(*pprev);
2702 if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2703 {
2704 --end;
2705 if (pcurr != pend) // GAP merge: 2 GAPS to be deleted
2706 {
2707 ++pcurr;
2708 copy_gaps:
2709 --end;
2710 do { *pprev++ = *pcurr++; } while (pcurr < pend);
2711 }
2712 }
2713 }
2714 else
2715 if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2716 {
2717 --(*pcurr);
2718 end += (pcurr == pend);
2719 }
2720 else // Worst case: split current GAP
2721 {
2722 if (*pcurr != bm::gap_max_bits-1) // last gap does not need memmove
2723 ::memmove(pcurr+2, pcurr, (end - curr + 1)*(sizeof(T)));
2724 end += 2;
2725 pcurr[0] = (T)(pos-1);
2726 pcurr[1] = (T)pos;
2727 }
2728
2729 // Set correct length word and last border word
2730 *buf = (T)((*buf & 7) + (end << 3));
2731 buf[end] = bm::gap_max_bits-1;
2732 return end;
2733 }
2734
2735 /*!
2736 \brief Sets or clears bit in the GAP buffer.
2737
2738 \param val - new bit value
2739 \param buf - GAP buffer.
2740 \param pos - Index of bit to set.
2741
2742 \return New GAP buffer length.
2743
2744 @ingroup gapfunc
2745 */
2746 template<typename T>
2747 unsigned gap_set_value(unsigned val,
2748 T* BMRESTRICT buf,
2749 unsigned pos) BMNOEXCEPT
2750 {
2751 BM_ASSERT(pos < bm::gap_max_bits);
2752 unsigned is_set;
2753 unsigned curr = bm::gap_bfind(buf, pos, &is_set);
2754 T end = (T)(*buf >> 3);
2755 if (is_set == val)
2756 return end;
2757
2758 T* pcurr = buf + curr;
2759 T* pprev = pcurr - 1;
2760 T* pend = buf + end;
2761
2762 // Special case, first bit GAP operation. There is no platform beside it.
2763 // initial flag must be inverted.
2764 if (!pos)
2765 {
2766 *buf ^= 1;
2767 if (buf[1]) // We need to insert a 1 bit GAP here
2768 {
2769 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2770 buf[1] = 0;
2771 ++end;
2772 }
2773 else // Only 1 bit in the GAP. We need to delete the first GAP.
2774 {
2775 pprev = buf + 1; pcurr = pprev + 1;
2776 goto copy_gaps;
2777 }
2778 }
2779 else
2780 if (curr > 1 && ((unsigned)(*pprev))+1 == pos) // Left border bit
2781 {
2782 ++(*pprev);
2783 if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2784 {
2785 --end;
2786 if (pcurr != pend) // GAP merge: 2 GAPS to be deleted
2787 {
2788 ++pcurr;
2789 copy_gaps:
2790 --end;
2791 do { *pprev++ = *pcurr++; } while (pcurr < pend);
2792 }
2793 }
2794 }
2795 else
2796 if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2797 {
2798 --(*pcurr);
2799 end += (pcurr == pend);
2800 }
2801 else // Worst case: split current GAP
2802 {
2803 if (*pcurr != bm::gap_max_bits-1) // last gap does not need memmove
2804 ::memmove(pcurr+2, pcurr, (end - curr + 1)*(sizeof(T)));
2805 end += 2;
2806 pcurr[0] = (T)(pos-1);
2807 pcurr[1] = (T)pos;
2808 }
2809
2810 // Set correct length word and last border word
2811 *buf = (T)((*buf & 7) + (end << 3));
2812 buf[end] = bm::gap_max_bits-1;
2813 return end;
2814 }
2815
2816 /*!
2817 \brief Add new value to the end of GAP buffer.
2818
2819 \param buf - GAP buffer.
2820 \param pos - Index of bit to set.
2821
2822 \return New GAP buffer length.
2823
2824 @ingroup gapfunc
2825 */
2826 template<typename T>
2827 unsigned gap_add_value(T* buf, unsigned pos) BMNOEXCEPT
2828 {
2829 BM_ASSERT(pos < bm::gap_max_bits);
2830
2831 T end = (T)(*buf >> 3);
2832 T curr = end;
2833 T* pcurr = buf + end;
2834 T* pend = pcurr;
2835 T* pprev = pcurr - 1;
2836
2837 // Special case, first bit GAP operation. There is no platform beside it.
2838 // initial flag must be inverted.
2839 if (!pos)
2840 {
2841 *buf ^= 1;
2842 if ( buf[1] ) // We need to insert a 1 bit platform here.
2843 {
2844 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2845 buf[1] = 0;
2846 ++end;
2847 }
2848 else // Only 1 bit in the GAP. We need to delete the first GAP.
2849 {
2850 pprev = buf + 1; pcurr = pprev + 1;
2851 --end;
2852 do { *pprev++ = *pcurr++; } while (pcurr < pend);
2853 }
2854 }
2855 else if (((unsigned)(*pprev))+1 == pos && (curr > 1) ) // Left border bit
2856 {
2857 ++(*pprev);
2858 if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2859 {
2860 --end;
2861 BM_ASSERT(pcurr == pend);
2862 }
2863 }
2864 else if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2865 {
2866 --(*pcurr);
2867 end += (pcurr == pend);
2868 }
2869 else // Worst case we need to split current block.
2870 {
2871 pcurr[0] = (T)(pos-1);
2872 pcurr[1] = (T)pos;
2873 end = (T)(end+2);
2874 }
2875
2876 // Set correct length word.
2877 *buf = (T)((*buf & 7) + (end << 3));
2878 buf[end] = bm::gap_max_bits - 1;
2879 return end;
2880 }
2881
2882 #ifdef __GNUG__
2883 #pragma GCC diagnostic pop
2884 #endif
2885
2886
2887 /*!
2888 @brief Right shift GAP block by 1 bit
2889 @param buf - block pointer
2890 @param co_flag - carry over from the previous block
2891 @param new_len - output length of the GAP block after the operation
2892
2893 @return carry over bit (1 or 0)
2894 @ingroup gapfunc
2895 */
2896 template<typename T>
2897 bool gap_shift_r1(T* BMRESTRICT buf,
2898 unsigned co_flag, unsigned* BMRESTRICT new_len) BMNOEXCEPT
2899 {
2900 BM_ASSERT(new_len);
2901 bool co;
2902 // 1: increment all GAP values by 1
2903 {
2904 unsigned bitval = *buf & 1;
2905 if (buf[1] == bm::gap_max_bits-1) // full GAP block
2906 {
2907 co = bitval;
2908 }
2909 else
2910 {
2911 unsigned len = (*buf >> 3);
2912 unsigned i = 1;
2913 for (; i < len; ++i)
2914 {
2915 buf[i]++;
2916 bitval ^= 1;
2917 } // for i
2918 BM_ASSERT(buf[i] == bm::gap_max_bits-1);
2919 if (buf[i-1] == bm::gap_max_bits-1) // last element shifts out
2920 {
2921 // Set correct length word
2922 --len;
2923 *buf = (T)((*buf & 7) + (len << 3));
2924 }
2925 co = bitval;
2926 }
2927 }
2928 // set bit position 0 with carry-in flag
2929 {
2930 unsigned is_set;
2931 *new_len = bm::gap_set_value(co_flag, buf, 0, &is_set);
2932 }
2933 return co;
2934 }
2935
2936 /*!
2937 @brief Left shift GAP block by 1 bit
2938 @param buf - block pointer
2939 @param co_flag - carry over from the previous block
2940 @param new_len - new length of the block
2941
2942 @return carry over bit (1 or 0)
2943 @ingroup gapfunc
2944 */
2945 template<typename T>
2946 bool gap_shift_l1(T* BMRESTRICT buf,
2947 unsigned co_flag, unsigned* BMRESTRICT new_len) BMNOEXCEPT
2948 {
2949 BM_ASSERT(new_len);
2950 unsigned is_set;
2951
2952 // 1: decrement all GAP values by 1
2953 //
2954 unsigned bitval = *buf & 1;
2955 bool co0 = bitval;
2956
2957 if (!buf[1]) // cannot decrement (corner case)
2958 {
2959 bitval ^= 1;
2960 *new_len = bm::gap_set_value(bitval, buf, 0, &is_set);
2961
2962 BM_ASSERT(is_set);
2963 BM_ASSERT(buf[1]);
2964 BM_ASSERT(bitval == unsigned(*buf & 1u));
2965
2966 if (*new_len == 1)
2967 {
2968 *new_len = bm::gap_set_value(co_flag, buf,
2969 bm::gap_max_bits-1, &is_set);
2970 return co0;
2971 }
2972 }
2973 if (buf[1] != bm::gap_max_bits-1) // full GAP block
2974 {
2975 BM_ASSERT(buf[1]);
2976 unsigned len = (*buf >> 3);
2977 unsigned i = 1;
2978 for (; i < len; ++i)
2979 {
2980 buf[i]--;
2981 bitval ^= 1;
2982 } // for i
2983 BM_ASSERT(buf[i] == bm::gap_max_bits-1);
2984 }
2985 // 2: set last bit position with carry-in flag
2986 //
2987 *new_len = bm::gap_set_value(co_flag, buf, bm::gap_max_bits-1, &is_set);
2988 return co0;
2989 }
2990
2991
2992 /*!
2993 \brief Convert array to GAP buffer.
2994
2995 \param buf - GAP buffer.
2996 \param arr - array of values to set
2997 \param len - length of the array
2998
2999 \return New GAP buffer length.
3000
3001 @ingroup gapfunc
3002 */
3003
3004 template<typename T>
3005 unsigned gap_set_array(T* buf, const T* arr, unsigned len) BMNOEXCEPT
3006 {
3007 *buf = (T)((*buf & 6u) + (1u << 3)); // gap header setup
3008
3009 T* pcurr = buf + 1;
3010
3011 unsigned i = 0;
3012 T curr = arr[i];
3013 if (curr != 0) // need to add the first gap: (0 to arr[0]-1)
3014 {
3015 *pcurr = (T)(curr - 1);
3016 ++pcurr;
3017 }
3018 else
3019 {
3020 ++(*buf); // GAP starts with 1
3021 }
3022 T prev = curr;
3023 T acc = prev;
3024
3025 for (i = 1; i < len; ++i)
3026 {
3027 curr = arr[i];
3028 if (curr == prev + 1)
3029 {
3030 ++acc;
3031 prev = curr;
3032 }
3033 else
3034 {
3035 *pcurr++ = acc;
3036 acc = curr;
3037 *pcurr++ = (T)(curr-1);
3038 }
3039 prev = curr;
3040 }
3041 *pcurr = acc;
3042 if (acc != bm::gap_max_bits - 1)
3043 {
3044 ++pcurr;
3045 *pcurr = bm::gap_max_bits - 1;
3046 }
3047
3048 unsigned gap_len = unsigned(pcurr - buf);
3049 BM_ASSERT(gap_len == ((gap_len << 3) >> 3));
3050
3051 *buf = (T)((*buf & 7) + (gap_len << 3));
3052 return gap_len+1;
3053 }
3054
3055
3056 //------------------------------------------------------------------------
3057
3058 /**
3059 \brief Compute number of GAPs in bit-array
3060 \param arr - array of BITs
3061 \param len - array length
3062
3063 @ingroup gapfunc
3064 */
3065 template<typename T>
3066 unsigned bit_array_compute_gaps(const T* arr, unsigned len) BMNOEXCEPT
3067 {
3068 unsigned gap_count = 1;
3069 T prev = arr[0];
3070 if (prev > 0)
3071 ++gap_count;
3072 for (unsigned i = 1; i < len; ++i)
3073 {
3074 T curr = arr[i];
3075 if (curr != prev + 1)
3076 {
3077 gap_count += 2;
3078 }
3079 prev = curr;
3080 }
3081 return gap_count;
3082 }
3083
3084
3085 //------------------------------------------------------------------------
3086
3087 /**
3088 \brief Searches for the next 1 bit in the GAP block
3089 \param buf - GAP buffer
3090 \param nbit - bit index to start checking from.
3091 \param prev - returns previously checked value
3092
3093 \return 0 if not found
3094
3095 @ingroup gapfunc
3096 */
3097 template<typename T>
3098 unsigned gap_block_find(const T* BMRESTRICT buf,
3099 unsigned nbit,
3100 bm::id_t* BMRESTRICT prev) BMNOEXCEPT
3101 {
3102 BM_ASSERT(nbit < bm::gap_max_bits);
3103
3104 unsigned bitval;
3105 unsigned gap_idx = bm::gap_bfind(buf, nbit, &bitval);
3106
3107 if (bitval) // positive block.
3108 {
3109 *prev = nbit;
3110 return 1u;
3111 }
3112 unsigned val = buf[gap_idx] + 1;
3113 *prev = val;
3114 return (val != bm::gap_max_bits); // no bug here.
3115 }
3116
3117 //------------------------------------------------------------------------
3118
3119
3120 /*!
3121 \brief Set 1 bit in a block
3122 @ingroup bitfunc
3123 */
3124 BMFORCEINLINE
3125 void set_bit(unsigned* dest, unsigned bitpos) BMNOEXCEPT
3126 {
3127 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3128 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3129 nbit &= bm::set_word_mask;
3130 dest[nword] |= unsigned(1u << nbit);
3131 }
3132
3133 /*!
3134 \brief Set 1 bit in a block
3135 @ingroup bitfunc
3136 */
3137 BMFORCEINLINE
3138 void clear_bit(unsigned* dest, unsigned bitpos) BMNOEXCEPT
3139 {
3140 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3141 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3142 nbit &= bm::set_word_mask;
3143 dest[nword] &= ~(unsigned(1u << nbit));
3144 }
3145
3146 /*!
3147 \brief Test 1 bit in a block
3148
3149 @ingroup bitfunc
3150 */
3151 BMFORCEINLINE
3152 unsigned test_bit(const unsigned* block, unsigned bitpos) BMNOEXCEPT
3153 {
3154 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3155 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3156 nbit &= bm::set_word_mask;
3157 return (block[nword] >> nbit) & 1u;
3158 }
3159
3160
3161 /*!
3162 \brief Sets bits to 1 in the bitblock.
3163 \param dest - Bitset buffer.
3164 \param bitpos - Offset of the start bit.
3165 \param bitcount - number of bits to set.
3166
3167 @ingroup bitfunc
3168 */
3169 inline
3170 void or_bit_block(unsigned* dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
3171 {
3172 const unsigned maskFF = ~0u;
3173
3174 dest += unsigned(bitpos >> bm::set_word_shift); // nword
3175 bitpos &= bm::set_word_mask;
3176
3177 if (bitcount == 1u) // special case (only 1 bit to set)
3178 {
3179 *dest |= (1u << bitpos);
3180 return;
3181 }
3182
3183 if (bitpos) // starting pos is not aligned
3184 {
3185 unsigned mask_r = maskFF << bitpos;
3186 unsigned right_margin = bitpos + bitcount;
3187 if (right_margin < 32)
3188 {
3189 *dest |= (maskFF >> (32 - right_margin)) & mask_r;
3190 return;
3191 }
3192 *dest++ |= mask_r;
3193 bitcount -= 32 - bitpos;
3194 }
3195 for ( ;bitcount >= 64; bitcount-=64, dest+=2)
3196 dest[0] = dest[1] = maskFF;
3197 if (bitcount >= 32)
3198 {
3199 *dest++ = maskFF; bitcount -= 32;
3200 }
3201 if (bitcount)
3202 {
3203 *dest |= maskFF >> (32 - bitcount);
3204 }
3205 }
3206
3207
3208 /*!
3209 \brief SUB (AND NOT) bit interval to 1 in the bitblock.
3210 \param dest - Bitset buffer.
3211 \param bitpos - Offset of the start bit.
3212 \param bitcount - number of bits to set.
3213
3214 @ingroup bitfunc
3215 */
3216 inline
3217 void sub_bit_block(unsigned* dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
3218 {
3219 const unsigned maskFF = ~0u;
3220
3221 dest += unsigned(bitpos >> bm::set_word_shift); // nword
3222 bitpos &= bm::set_word_mask;
3223
3224 if (bitcount == 1u) // special case (only 1 bit to set)
3225 {
3226 *dest &= ~(1u << bitpos);
3227 return;
3228 }
3229
3230 if (bitpos) // starting pos is not aligned
3231 {
3232 unsigned mask_r = maskFF << bitpos;
3233 unsigned right_margin = bitpos + bitcount;
3234 if (right_margin < 32)
3235 {
3236 *dest &= ~((maskFF >> (32 - right_margin)) & mask_r);
3237 return;
3238 }
3239 *dest++ &= ~mask_r;
3240 bitcount -= 32 - bitpos;
3241 }
3242 for ( ;bitcount >= 64; bitcount-=64, dest+=2)
3243 dest[0] = dest[1] = 0u;
3244 if (bitcount >= 32)
3245 {
3246 *dest++ = 0u; bitcount -= 32;
3247 }
3248 if (bitcount)
3249 {
3250 *dest &= ~(maskFF >> (32 - bitcount));
3251 }
3252 }
3253
3254
3255
3256 /*!
3257 \brief XOR bit interval to 1 in the bitblock.
3258 \param dest - Bitset buffer.
3259 \param bitpos - Offset of the start bit.
3260 \param bitcount - number of bits to set.
3261
3262 @ingroup bitfunc
3263 */
3264 inline void xor_bit_block(unsigned* dest,
3265 unsigned bitpos,
3266 unsigned bitcount) BMNOEXCEPT
3267 {
3268 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3269 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3270 nbit &= bm::set_word_mask;
3271
3272 bm::word_t* word = dest + nword;
3273
3274 if (bitcount == 1) // special case (only 1 bit to set)
3275 {
3276 *word ^= unsigned(1 << nbit);
3277 return;
3278 }
3279
3280 if (nbit) // starting position is not aligned
3281 {
3282 unsigned right_margin = nbit + bitcount;
3283
3284 // here we checking if we setting bits only in the current
3285 // word. Example: 00111000000000000000000000000000 (32 bits word)
3286
3287 if (right_margin < 32)
3288 {
3289 unsigned mask =
3290 bm::block_set_table<true>::_right[nbit] &
3291 bm::block_set_table<true>::_left[right_margin-1];
3292 *word ^= mask;
3293 return;
3294 }
3295 *word ^= block_set_table<true>::_right[nbit];
3296 bitcount -= 32 - nbit;
3297 ++word;
3298 }
3299 for ( ;bitcount >= 64; bitcount-=64, word+=2)
3300 {
3301 word[0] ^= ~0u; word[1] ^= ~0u;
3302 }
3303 if (bitcount >= 32)
3304 {
3305 *word++ ^= ~0u; bitcount -= 32;
3306 }
3307 if (bitcount)
3308 *word ^= block_set_table<true>::_left[bitcount-1];
3309 }
3310
3311
3312 /*!
3313 \brief SUB (AND NOT) GAP block to bitblock.
3314 \param dest - bitblock buffer pointer.
3315 \param pcurr - GAP buffer pointer.
3316
3317 @ingroup gapfunc
3318 */
3319 template<typename T>
3320 void gap_sub_to_bitset(unsigned* BMRESTRICT dest,
3321 const T* BMRESTRICT pcurr) BMNOEXCEPT
3322 {
3323 BM_ASSERT(dest && pcurr);
3324
3325 const T* pend = pcurr + (*pcurr >> 3);
3326 if (*pcurr & 1) // Starts with 1
3327 {
3328 bm::sub_bit_block(dest, 0, 1 + pcurr[1]);
3329 ++pcurr;
3330 }
3331 for (pcurr += 2; pcurr <= pend; pcurr += 2)
3332 {
3333 BM_ASSERT(*pcurr > pcurr[-1]);
3334 bm::sub_bit_block(dest, 1 + pcurr[-1], *pcurr - pcurr[-1]);
3335 }
3336 }
3337
3338
3339 /*!
3340 \brief SUB (AND NOT) GAP block to bitblock with digest assist
3341
3342 \param dest - bitblock buffer pointer.
3343 \param pcurr - GAP buffer pointer.
3344 \param digest0 - digest of 0 strides inside bit block
3345
3346 @ingroup gapfunc
3347 */
3348 template<typename T>
3349 void gap_sub_to_bitset(unsigned* BMRESTRICT dest,
3350 const T* BMRESTRICT pcurr, bm::id64_t digest0) BMNOEXCEPT
3351 {
3352 BM_ASSERT(dest && pcurr);
3353
3354 const T* pend = pcurr + (*pcurr >> 3);
3355 if (*pcurr & 1) // Starts with 1
3356 {
3357 bool all_zero = bm::check_zero_digest(digest0, 0, pcurr[1]+1);
3358 if (!all_zero)
3359 bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3360 pcurr += 3;
3361 }
3362 else
3363 pcurr += 2;
3364
3365 // wind forward to digest start
3366 {
3367 unsigned tz = bm::count_trailing_zeros_u64(digest0);
3368 unsigned start_pos = tz << set_block_digest_pos_shift;
3369 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0"
3370 {
3371 if (*pcurr >= start_pos)
3372 break;
3373 }
3374 }
3375
3376 unsigned lz = bm::count_leading_zeros_u64(digest0);
3377 unsigned stop_pos = (64u - lz) << set_block_digest_pos_shift;
3378
3379 unsigned bc, pos;
3380 T prev;
3381 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "1" again
3382 {
3383 BM_ASSERT(*pcurr > *(pcurr-1));
3384 prev = pcurr[-1];
3385 bc = *pcurr - prev;
3386 pos = 1u + prev;
3387
3388 bool all_zero = bm::check_zero_digest(digest0, prev, *pcurr);
3389 if (!all_zero)
3390 bm::sub_bit_block(dest, pos, bc);
3391
3392 if (pos > stop_pos)
3393 break; // early break is possible based on digest tail
3394
3395 } // for
3396 }
3397
3398
3399
3400 /*!
3401 \brief XOR GAP block to bitblock.
3402 \param dest - bitblock buffer pointer.
3403 \param pcurr - GAP buffer pointer.
3404
3405 @ingroup gapfunc
3406 */
3407 template<typename T>
3408 void gap_xor_to_bitset(unsigned* BMRESTRICT dest,
3409 const T* BMRESTRICT pcurr) BMNOEXCEPT
3410 {
3411 BM_ASSERT(dest && pcurr);
3412
3413 const T* pend = pcurr + (*pcurr >> 3);
3414 if (*pcurr & 1) // Starts with 1
3415 {
3416 bm::xor_bit_block(dest, 0, 1 + pcurr[1]);
3417 ++pcurr;
3418 }
3419 for (pcurr += 2; pcurr <= pend; pcurr += 2)
3420 {
3421 BM_ASSERT(*pcurr > pcurr[-1]);
3422 bm::xor_bit_block(dest, 1 + pcurr[-1], *pcurr - pcurr[-1]);
3423 }
3424 }
3425
3426
3427 /*!
3428 \brief Adds(OR) GAP block to bitblock.
3429 \param dest - bitblock buffer pointer.
3430 \param pcurr - GAP buffer pointer.
3431 \param len - gap length
3432
3433 @ingroup gapfunc
3434 */
3435 template<typename T>
3436 void gap_add_to_bitset(unsigned* BMRESTRICT dest,
3437 const T* BMRESTRICT pcurr, unsigned len) BMNOEXCEPT
3438 {
3439 BM_ASSERT(dest && pcurr);
3440
3441 const T* pend = pcurr + len;
3442 if (*pcurr & 1) // Starts with 1
3443 {
3444 bm::or_bit_block(dest, 0, 1 + pcurr[1]);
3445 pcurr += 3;
3446 }
3447 else
3448 pcurr += 2;
3449
3450 unsigned bc, pos;
3451 for (; pcurr <= pend; )
3452 {
3453 BM_ASSERT(*pcurr > pcurr[-1]);
3454 pos = 1u + pcurr[-1];
3455 bc = *pcurr - pcurr[-1];
3456 pcurr += 2;
3457 bm::or_bit_block(dest, pos, bc);
3458 }
3459 }
3460
3461
3462 /*!
3463 \brief Adds(OR) GAP block to bitblock.
3464 \param dest - bitblock buffer pointer.
3465 \param pcurr - GAP buffer pointer.
3466
3467 @ingroup gapfunc
3468 */
3469 template<typename T>
3470 void gap_add_to_bitset(unsigned* BMRESTRICT dest,
3471 const T* BMRESTRICT pcurr) BMNOEXCEPT
3472 {
3473 unsigned len = (*pcurr >> 3);
3474 gap_add_to_bitset(dest, pcurr, len);
3475 }
3476
3477
3478 /*!
3479 \brief ANDs GAP block to bitblock.
3480 \param dest - bitblock buffer pointer.
3481 \param pcurr - GAP buffer pointer.
3482
3483 @ingroup gapfunc
3484 */
3485 template<typename T>
3486 void gap_and_to_bitset(unsigned* BMRESTRICT dest,
3487 const T* BMRESTRICT pcurr) BMNOEXCEPT
3488 {
3489 BM_ASSERT(dest && pcurr);
3490
3491 const T* pend = pcurr + (*pcurr >> 3);
3492 if (!(*pcurr & 1) ) // Starts with 0
3493 {
3494 bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3495 pcurr += 3;
3496 }
3497 else
3498 pcurr += 2;
3499
3500 unsigned bc, pos;
3501 for (; pcurr <= pend; ) // now we are in GAP "0" again
3502 {
3503 BM_ASSERT(*pcurr > *(pcurr-1));
3504 pos = 1u + pcurr[-1];
3505 bc = *pcurr - pcurr[-1];
3506 pcurr += 2;
3507 bm::sub_bit_block(dest, pos, bc);
3508 }
3509 }
3510
3511
3512 /*!
3513 \brief ANDs GAP block to bitblock with digest assist
3514 \param dest - bitblock buffer pointer.
3515 \param pcurr - GAP buffer pointer.
3516 \param digest0 - digest of 0 strides for the destination
3517
3518 @ingroup gapfunc
3519 */
3520 template<typename T>
3521 void gap_and_to_bitset(unsigned* BMRESTRICT dest,
3522 const T* BMRESTRICT pcurr, bm::id64_t digest0) BMNOEXCEPT
3523 {
3524 BM_ASSERT(dest && pcurr);
3525 if (!digest0)
3526 return;
3527
3528 const T* pend = pcurr + (*pcurr >> 3);
3529 if (!(*pcurr & 1) ) // Starts with 0
3530 {
3531 bool all_zero = bm::check_zero_digest(digest0, 0, pcurr[1]+1);
3532 if (!all_zero)
3533 bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3534 pcurr += 3;
3535 }
3536 else
3537 pcurr += 2;
3538
3539 // wind forward to digest start
3540 {
3541 unsigned tz = bm::count_trailing_zeros_u64(digest0);
3542 unsigned start_pos = tz << set_block_digest_pos_shift;
3543 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0"
3544 {
3545 if (*pcurr >= start_pos)
3546 break;
3547 }
3548 }
3549
3550 unsigned lz = bm::count_leading_zeros_u64(digest0);
3551 unsigned stop_pos = (64u - lz) << set_block_digest_pos_shift;
3552
3553 unsigned bc, pos;
3554 T prev;
3555 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0" again
3556 {
3557 BM_ASSERT(*pcurr > *(pcurr-1));
3558
3559 prev = pcurr[-1];
3560 bc = *pcurr - prev;
3561 pos = 1u + prev;
3562
3563 bool all_zero = bm::check_zero_digest(digest0, prev, *pcurr);
3564 if (!all_zero)
3565 bm::sub_bit_block(dest, pos, bc);
3566
3567 if (pos > stop_pos) // early break is possible based on digest tail
3568 break;
3569
3570 } // for
3571 }
3572
3573
3574 /*!
3575 \brief Compute bitcount of bit block AND masked by GAP block
3576 \param block - bitblock buffer pointer
3577 \param pcurr - GAP buffer pointer
3578 \return bitcount - cardinality of the AND product
3579
3580 @ingroup gapfunc
3581 */
3582 template<typename T>
3583 bm::id_t gap_bitset_and_count(const unsigned* BMRESTRICT block,
3584 const T* BMRESTRICT pcurr) BMNOEXCEPT
3585 {
3586 BM_ASSERT(block);
3587 const T* pend = pcurr + (*pcurr >> 3);
3588 bm::id_t count = 0;
3589 if (*pcurr & 1) // Starts with 1
3590 {
3591 count += bm::bit_block_calc_count_range(block, 0, pcurr[1]);
3592 ++pcurr;
3593 }
3594 for (pcurr +=2 ;pcurr <= pend; pcurr += 2)
3595 {
3596 count += bm::bit_block_calc_count_range(block, pcurr[-1]+1, *pcurr);
3597 }
3598 return count;
3599 }
3600
3601
3602 /*!
3603 \brief Bitcount test of bit block AND masked by GAP block.
3604 \param block - bitblock buffer pointer
3605 \param pcurr - GAP buffer pointer
3606 \return non-zero value if AND produces any result
3607
3608 @ingroup gapfunc
3609 */
3610 template<typename T>
3611 bm::id_t gap_bitset_and_any(const unsigned* BMRESTRICT block,
3612 const T* BMRESTRICT pcurr) BMNOEXCEPT
3613 {
3614 BM_ASSERT(block);
3615
3616 const T* pend = pcurr + (*pcurr >> 3);
3617 bm::id_t count = 0;
3618 if (*pcurr & 1) // Starts with 1
3619 {
3620 count = bm::bit_block_any_range(block, 0, pcurr[1]);
3621 ++pcurr;
3622 }
3623 for (pcurr +=2 ;!count && pcurr <= pend; pcurr += 2)
3624 {
3625 count = bm::bit_block_any_range(block, pcurr[-1]+1, *pcurr);
3626 }
3627 return count;
3628 }
3629
3630
3631
3632 /*!
3633 \brief Compute bitcount of bit block SUB masked by GAP block.
3634 \param block - bitblock buffer pointer.
3635 \param buf - GAP buffer pointer.
3636 \return bit-count result of AND NOT operation
3637
3638 @ingroup gapfunc
3639 */
3640 template<typename T>
3641 bm::id_t gap_bitset_sub_count(const unsigned* BMRESTRICT block,
3642 const T* BMRESTRICT buf) BMNOEXCEPT
3643 {
3644 BM_ASSERT(block);
3645
3646 const T* pcurr = buf;
3647 const T* pend = pcurr + (*pcurr >> 3);
3648 ++pcurr;
3649
3650 bm::id_t count = 0;
3651
3652 if (!(*buf & 1)) // Starts with 0
3653 {
3654 count += bit_block_calc_count_range(block, 0, *pcurr);
3655 ++pcurr;
3656 }
3657 ++pcurr; // now we are in GAP "0" again
3658
3659 for (;pcurr <= pend; pcurr+=2)
3660 {
3661 count += bm::bit_block_calc_count_range(block, *(pcurr-1)+1, *pcurr);
3662 }
3663 return count;
3664 }
3665
3666
3667 /*!
3668 \brief Compute bitcount test of bit block SUB masked by GAP block
3669 \param block - bitblock buffer pointer
3670 \param buf - GAP buffer pointer
3671 \return non-zero value if AND NOT produces any 1 bits
3672
3673 @ingroup gapfunc
3674 */
3675 template<typename T>
3676 bm::id_t gap_bitset_sub_any(const unsigned* BMRESTRICT block,
3677 const T* BMRESTRICT buf) BMNOEXCEPT
3678 {
3679 BM_ASSERT(block);
3680
3681 const T* pcurr = buf;
3682 const T* pend = pcurr + (*pcurr >> 3);
3683 ++pcurr;
3684
3685 bm::id_t count = 0;
3686
3687 if (!(*buf & 1)) // Starts with 0
3688 {
3689 count += bit_block_any_range(block, 0, *pcurr);
3690 if (count)
3691 return count;
3692 ++pcurr;
3693 }
3694 ++pcurr; // now we are in GAP "0" again
3695
3696 for (; !count && pcurr <= pend; pcurr+=2)
3697 {
3698 count += bm::bit_block_any_range(block, *(pcurr-1)+1, *pcurr);
3699 }
3700 return count;
3701 }
3702
3703
3704
3705 /*!
3706 \brief Compute bitcount of bit block XOR masked by GAP block
3707 \param block - bitblock buffer pointer
3708 \param buf - GAP buffer pointer
3709 \return bit count value of XOR operation
3710
3711 @ingroup gapfunc
3712 */
3713 template<typename T>
3714 bm::id_t gap_bitset_xor_count(const unsigned* BMRESTRICT block,
3715 const T* BMRESTRICT buf) BMNOEXCEPT
3716 {
3717 BM_ASSERT(block);
3718
3719 const T* pcurr = buf;
3720 const T* pend = pcurr + (*pcurr >> 3);
3721 ++pcurr;
3722
3723 unsigned bitval = *buf & 1;
3724
3725 bm::id_t count = bm::bit_block_calc_count_range(block, 0, *pcurr);
3726 if (bitval)
3727 {
3728 count = *pcurr + 1 - count;
3729 }
3730
3731 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
3732 {
3733 T prev = (T)(*(pcurr-1)+1);
3734 bm::id_t c = bit_block_calc_count_range(block, prev, *pcurr);
3735
3736 if (bitval) // 1 gap; means Result = Total_Bits - BitCount;
3737 c = (*pcurr - prev + 1) - c;
3738 count += c;
3739 }
3740 return count;
3741 }
3742
3743 /*!
3744 \brief Compute bitcount test of bit block XOR masked by GAP block.
3745 \param block - bitblock buffer pointer
3746 \param buf - GAP buffer pointer
3747 \return non-zero value if XOR returns anything
3748
3749 @ingroup gapfunc
3750 */
3751 template<typename T>
3752 bm::id_t gap_bitset_xor_any(const unsigned* BMRESTRICT block,
3753 const T* BMRESTRICT buf) BMNOEXCEPT
3754 {
3755 BM_ASSERT(block);
3756
3757 const T* pcurr = buf;
3758 const T* pend = pcurr + (*pcurr >> 3);
3759 ++pcurr;
3760
3761 unsigned bitval = *buf & 1;
3762
3763 bm::id_t count = bit_block_any_range(block, 0, *pcurr);
3764 if (bitval)
3765 count = *pcurr + 1 - count;
3766
3767 for (bitval^=1, ++pcurr; !count && pcurr <= pend; bitval^=1, ++pcurr)
3768 {
3769 T prev = (T)(*(pcurr-1)+1);
3770 bm::id_t c = bit_block_any_range(block, prev, *pcurr);
3771
3772 if (bitval) // 1 gap; means Result = Total_Bits - BitCount;
3773 c = (*pcurr - prev + 1) - c;
3774 count += c;
3775 }
3776 return count;
3777 }
3778
3779
3780
3781 /*!
3782 \brief Compute bitcount of bit block OR masked by GAP block.
3783 \param block - bitblock buffer pointer.
3784 \param buf - GAP buffer pointer.
3785 \return bit count of OR
3786
3787 @ingroup gapfunc
3788 */
3789 template<typename T>
3790 bm::id_t gap_bitset_or_count(const unsigned* BMRESTRICT block,
3791 const T* BMRESTRICT buf) BMNOEXCEPT
3792 {
3793 BM_ASSERT(block);
3794 const T* pcurr = buf;
3795 const T* pend = pcurr + (*pcurr >> 3);
3796 ++pcurr;
3797
3798 unsigned bitval = *buf & 1;
3799
3800 bm::id_t count = bitval ? *pcurr + 1
3801 : bm::bit_block_calc_count_range(block, 0, *pcurr);
3802 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
3803 {
3804 T prev = (T)(*(pcurr-1)+1);
3805 bm::id_t c =
3806 bitval ? (*pcurr - prev + 1)
3807 : bm::bit_block_calc_count_range(block, prev, *pcurr);
3808 count += c;
3809 }
3810 return count;
3811 }
3812
3813 /*!
3814 \brief Compute bitcount test of bit block OR masked by GAP block
3815 \param block - bitblock buffer pointer
3816 \param buf - GAP buffer pointer
3817 \return non zero value if union (OR) returns anything
3818
3819 @ingroup gapfunc
3820 */
3821 template<typename T>
3822 bm::id_t gap_bitset_or_any(const unsigned* BMRESTRICT block,
3823 const T* BMRESTRICT buf) BMNOEXCEPT
3824 {
3825 bool b = !bm::gap_is_all_zero(buf) ||
3826 !bm::bit_is_all_zero(block);
3827 return b;
3828 }
3829
3830
3831
3832 /*!
3833 \brief Bitblock memset operation.
3834
3835 \param dst - destination block.
3836 \param value - value to set.
3837
3838 @ingroup bitfunc
3839 */
3840 inline
3841 void bit_block_set(bm::word_t* BMRESTRICT dst, bm::word_t value) BMNOEXCEPT
3842 {
3843 #ifdef BMVECTOPT
3844 VECT_SET_BLOCK(dst, value);
3845 #else
3846 ::memset(dst, int(value), bm::set_block_size * sizeof(bm::word_t));
3847 #endif
3848 }
3849
3850
3851 /*!
3852 \brief GAP block to bitblock conversion.
3853 \param dest - bitblock buffer pointer.
3854 \param buf - GAP buffer pointer.
3855
3856 @ingroup gapfunc
3857 */
3858 template<typename T>
3859 void gap_convert_to_bitset(unsigned* BMRESTRICT dest,
3860 const T* BMRESTRICT buf) BMNOEXCEPT
3861 {
3862 bm::bit_block_set(dest, 0);
3863 bm::gap_add_to_bitset(dest, buf);
3864 }
3865
3866
3867
3868 /*!
3869 \brief Smart GAP block to bitblock conversion.
3870
3871 Checks if GAP block is ALL-ZERO or ALL-ON. In those cases returns
3872 pointer on special static bitblocks.
3873
3874 \param dest - bitblock buffer pointer.
3875 \param buf - GAP buffer pointer.
3876 \param set_max - max possible bitset length
3877
3878 @ingroup gapfunc
3879 */
3880 template<typename T>
3881 unsigned* gap_convert_to_bitset_smart(unsigned* BMRESTRICT dest,
3882 const T* BMRESTRICT buf,
3883 id_t set_max) BMNOEXCEPT
3884 {
3885 if (buf[1] == set_max - 1)
3886 return (buf[0] & 1) ? FULL_BLOCK_REAL_ADDR : 0;
3887 bm::gap_convert_to_bitset(dest, buf);
3888 return dest;
3889 }
3890
3891
3892 /*!
3893 \brief Calculates sum of all words in GAP block. (For debugging purposes)
3894 \note For debugging and testing ONLY.
3895 \param buf - GAP buffer pointer.
3896 \return Sum of all words.
3897
3898 @ingroup gapfunc
3899 @internal
3900 */
3901 template<typename T>
3902 unsigned gap_control_sum(const T* buf) BMNOEXCEPT
3903 {
3904 unsigned end = *buf >> 3;
3905
3906 const T* pcurr = buf;
3907 const T* pend = pcurr + (*pcurr >> 3);
3908 ++pcurr;
3909
3910 if (*buf & 1) // Starts with 1
3911 {
3912 ++pcurr;
3913 }
3914 ++pcurr; // now we are in GAP "1" again
3915 while (pcurr <= pend)
3916 {
3917 BM_ASSERT(*pcurr > *(pcurr-1));
3918 pcurr += 2;
3919 }
3920 return buf[end];
3921 }
3922
3923
3924 /*!
3925 \brief Sets all bits to 0 or 1 (GAP)
3926 \param buf - GAP buffer pointer.
3927 \param set_max - max possible bitset length
3928 \param value - value to set
3929
3930 @ingroup gapfunc
3931 */
3932 template<class T>
3933 void gap_set_all(T* buf, unsigned set_max, unsigned value) BMNOEXCEPT
3934 {
3935 BM_ASSERT(value == 0 || value == 1);
3936 *buf = (T)((*buf & 6u) + (1u << 3) + value);
3937 *(++buf) = (T)(set_max - 1);
3938 }
3939
3940
3941 /*!
3942 \brief Init gap block so it has block in it (can be whole block)
3943 \param buf - GAP buffer pointer.
3944 \param from - one block start
3945 \param to - one block end
3946 \param value - (block value)1 or 0
3947
3948 @ingroup gapfunc
3949 */
3950 template<class T>
3951 void gap_init_range_block(T* buf,
3952 T from,
3953 T to,
3954 T value) BMNOEXCEPT
3955 {
3956 BM_ASSERT(value == 0 || value == 1);
3957 const unsigned set_max = bm::bits_in_block;
3958
3959 unsigned gap_len;
3960 if (from == 0)
3961 {
3962 if (to == set_max - 1)
3963 {
3964 bm::gap_set_all(buf, set_max, value);
3965 }
3966 else
3967 {
3968 gap_len = 2;
3969 buf[1] = (T)to;
3970 buf[2] = (T)(set_max - 1);
3971 buf[0] = (T)((*buf & 6u) + (gap_len << 3) + value);
3972 }
3973 return;
3974 }
3975 // from != 0
3976
3977 value = !value;
3978 if (to == set_max - 1)
3979 {
3980 gap_len = 2;
3981 buf[1] = (T)(from - 1);
3982 buf[2] = (T)(set_max - 1);
3983 }
3984 else
3985 {
3986 gap_len = 3;
3987 buf[1] = (T) (from - 1);
3988 buf[2] = (T) to;
3989 buf[3] = (T)(set_max - 1);
3990 }
3991 buf[0] = (T)((*buf & 6u) + (gap_len << 3) + value);
3992 }
3993
3994
3995 /*!
3996 \brief Inverts all bits in the GAP buffer.
3997 \param buf - GAP buffer pointer.
3998
3999 @ingroup gapfunc
4000 */
4001 template<typename T> void gap_invert(T* buf) BMNOEXCEPT
4002 {
4003 *buf ^= 1;
4004 }
4005
4006
4007 #ifdef __GNUG__
4008 #pragma GCC diagnostic push
4009 #pragma GCC diagnostic ignored "-Wconversion"
4010 #endif
4011
4012 /*!
4013 \brief Sets GAP block capacity level.
4014 \param buf - GAP buffer pointer.
4015 \param level new GAP block capacity level.
4016
4017 @ingroup gapfunc
4018 */
4019 template<typename T>
4020 void set_gap_level(T* buf, int level) BMNOEXCEPT
4021 {
4022 BM_ASSERT(level >= 0);
4023 BM_ASSERT(unsigned(level) < bm::gap_levels);
4024
4025 *buf = (T)(((level & 3) << 1) | (*buf & 1) | (*buf & ~7));
4026 }
4027 #ifdef __GNUG__
4028 #pragma GCC diagnostic pop
4029 #endif
4030
4031
4032
4033 /*!
4034 \brief Calculates GAP block capacity level.
4035 \param len - GAP buffer length.
4036 \param glevel_len - GAP lengths table
4037 \return GAP block capacity level.
4038 -1 if block does not fit any level.
4039 @ingroup gapfunc
4040 */
4041 template<typename T>
4042 int gap_calc_level(unsigned len, const T* glevel_len) BMNOEXCEPT
4043 {
4044 if (len <= unsigned(glevel_len[0]-4)) return 0;
4045 if (len <= unsigned(glevel_len[1]-4)) return 1;
4046 if (len <= unsigned(glevel_len[2]-4)) return 2;
4047 if (len <= unsigned(glevel_len[3]-4)) return 3;
4048
4049 BM_ASSERT(bm::gap_levels == 4);
4050 return -1;
4051 }
4052
4053 /*! @brief Returns number of free elements in GAP block array.
4054 Difference between GAP block capacity on this level and actual GAP length.
4055
4056 @param buf - GAP buffer pointer
4057 @param glevel_len - GAP lengths table
4058
4059 @return Number of free GAP elements
4060 @ingroup gapfunc
4061 */
4062 template<typename T>
4063 inline unsigned gap_free_elements(const T* BMRESTRICT buf,
4064 const T* BMRESTRICT glevel_len) BMNOEXCEPT
4065 {
4066 unsigned len = bm::gap_length(buf);
4067 unsigned capacity = bm::gap_capacity(buf, glevel_len);
4068 return capacity - len;
4069 }
4070
4071 /*!
4072 \brief Lexicographical comparison of BIT buffers.
4073 \param buf1 - First buffer pointer.
4074 \param buf2 - Second buffer pointer.
4075 \param len - Buffer length in elements (T).
4076 \return <0 - less, =0 - equal, >0 - greater.
4077
4078 @ingroup bitfunc
4079 */
4080 template<typename T>
4081 int bitcmp(const T* buf1, const T* buf2, unsigned len) BMNOEXCEPT
4082 {
4083 BM_ASSERT(len);
4084 const T* pend1 = buf1 + len;
4085 do
4086 {
4087 T w1 = *buf1++;
4088 T w2 = *buf2++;
4089 T diff = w1 ^ w2;
4090 if (diff)
4091 return (w1 & diff & -diff) ? 1 : -1;
4092 } while (buf1 < pend1);
4093 return 0;
4094 }
4095
4096 /*!
4097 \brief Find first bit which is different between two bit-blocks
4098 \param blk1 - block 1
4099 \param blk2 - block 2
4100 \param pos - out - position of difference (undefined if blocks are equal)
4101 \return true if difference was found
4102
4103 @ingroup bitfunc
4104 */
4105 inline
4106 bool bit_find_first_diff(const bm::word_t* BMRESTRICT blk1,
4107 const bm::word_t* BMRESTRICT blk2,
4108 unsigned* BMRESTRICT pos) BMNOEXCEPT
4109 {
4110 BM_ASSERT(blk1 && blk2 && pos);
4111 #ifdef VECT_BIT_FIND_DIFF
4112 bool f = VECT_BIT_FIND_DIFF(blk1, blk2, pos);
4113 return f;
4114 #else
4115 #ifdef BM64OPT
4116 BM_ASSERT(sizeof(bm::wordop_t) == 8);
4117
4118 const bm::wordop_t* b1 = (const bm::wordop_t*) blk1;
4119 const bm::wordop_t* b2 = (const bm::wordop_t*) blk2;
4120
4121 for (unsigned i = 0; i < bm::set_block_size/2; ++i)
4122 {
4123 bm::wordop_t w1 = b1[i]; bm::wordop_t w2 = b2[i];
4124 bm::wordop_t diff = w1 ^ w2;
4125 if (diff)
4126 {
4127 unsigned idx = bm::count_trailing_zeros_u64(diff);
4128 *pos = unsigned(idx + (i * 8u * unsigned(sizeof(bm::wordop_t))));
4129 return true;
4130 }
4131 } // for
4132 #else
4133 for (unsigned i = 0; i < bm::set_block_size; ++i)
4134 {
4135 bm::word_t w1 = blk1[i]; bm::word_t w2 = blk2[i];
4136 bm::word_t diff = w1 ^ w2;
4137 if (diff)
4138 {
4139 unsigned idx = bm::bit_scan_forward32(diff); // trailing zeros
4140 *pos = unsigned(idx + (i * 8u * sizeof(bm::word_t)));
4141 return true;
4142 }
4143 } // for
4144 #endif
4145 #endif
4146 return false;
4147 }
4148
4149
4150 #ifndef BMAVX2OPT
4151
4152 /*!
4153 \brief Converts bit block to GAP.
4154 \param dest - Destinatio GAP buffer.
4155 \param block - Source bitblock buffer.
4156 \param dest_len - length of the destination buffer.
4157 \return New length of GAP block or 0 if conversion failed
4158 (insufficicent space).
4159
4160 @ingroup gapfunc
4161 */
4162 inline
4163 unsigned bit_block_to_gap(gap_word_t* BMRESTRICT dest,
4164 const unsigned* BMRESTRICT block,
4165 unsigned dest_len) BMNOEXCEPT
4166 {
4167 const unsigned* BMRESTRICT block_end = block + bm::set_block_size;
4168 gap_word_t* BMRESTRICT pcurr = dest;
4169 gap_word_t* BMRESTRICT end = dest + dest_len; (void)end;
4170
4171 unsigned bitval = (*block) & 1u;
4172 *pcurr++ = bm::gap_word_t(bitval);
4173 *pcurr = 0;
4174 unsigned bit_idx = 0;
4175
4176 do
4177 {
4178 unsigned val = *block;
4179 while (!val || val == ~0u)
4180 {
4181 if (bitval != unsigned(bool(val)))
4182 {
4183 *pcurr++ = (gap_word_t)(bit_idx-1);
4184 bitval ^= 1u;
4185 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4186 BM_ASSERT(pcurr != end);
4187 }
4188 bit_idx += unsigned(sizeof(*block) * 8);
4189 if (++block >= block_end)
4190 goto complete;
4191 val = *block;
4192 } // while
4193
4194 // process "0100011" word
4195 //
4196 unsigned bits_consumed = 0;
4197 do
4198 {
4199 unsigned tz = 1u;
4200 if (bitval != (val & 1u))
4201 {
4202 *pcurr++ = (gap_word_t)(bit_idx-1);
4203 bitval ^= 1u;
4204 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4205 BM_ASSERT(pcurr != end);
4206 }
4207 else // match, find the next idx
4208 {
4209 tz = bm::bit_scan_forward32(bitval ? ~val : val);
4210 // possible alternative:
4211 // tz = bm::count_trailing_zeros(bitval ? ~val : val);
4212 }
4213
4214 bits_consumed += tz;
4215 bit_idx += tz;
4216 val >>= tz;
4217
4218 if (!val)
4219 {
4220 if (bits_consumed < 32u)
4221 {
4222 *pcurr++ = (gap_word_t)(bit_idx-1);
4223 bitval ^= 1u;
4224 bit_idx += 32u - bits_consumed;
4225 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4226 BM_ASSERT(pcurr != end);
4227 }
4228 break;
4229 }
4230 } while (1);
4231
4232 } while(++block < block_end);
4233
4234 complete:
4235 *pcurr = (gap_word_t)(bit_idx-1);
4236 unsigned len = (unsigned)(pcurr - dest);
4237 *dest = (gap_word_t)((*dest & 7) + (len << 3));
4238 return len;
4239 }
4240 #endif
4241
4242 /**
4243 Convert bit block to GAP representation
4244 @internal
4245 @ingroup bitfunc
4246 */
4247 inline
4248 unsigned bit_to_gap(gap_word_t* BMRESTRICT dest,
4249 const unsigned* BMRESTRICT block,
4250 unsigned dest_len) BMNOEXCEPT
4251 {
4252 #if defined(VECT_BIT_TO_GAP)
4253 return VECT_BIT_TO_GAP(dest, block, dest_len);
4254 #else
4255 return bm::bit_block_to_gap(dest, block, dest_len);
4256 #endif
4257 }
4258
4259
4260 /*!
4261 \brief Iterate gap block as delta-bits with a functor
4262 @ingroup gapfunc
4263 */
4264 template<class T, class F>
4265 void for_each_gap_dbit(const T* buf, F& func)
4266 {
4267 const T* pcurr = buf;
4268 const T* pend = pcurr + (*pcurr >> 3);
4269
4270 ++pcurr;
4271
4272 unsigned prev = 0;
4273 unsigned first_inc;
4274
4275 if (*buf & 1)
4276 {
4277 first_inc = 0;
4278 unsigned to = *pcurr;
4279 for (unsigned i = 0; i <= to; ++i)
4280 {
4281 func(1);
4282 }
4283 prev = to;
4284 ++pcurr;
4285 }
4286 else
4287 {
4288 first_inc = 1;
4289 }
4290 ++pcurr; // set GAP to 1
4291
4292 while (pcurr <= pend)
4293 {
4294 unsigned from = *(pcurr-1)+1;
4295 unsigned to = *pcurr;
4296 if (first_inc)
4297 {
4298 func(from - prev + first_inc);
4299 first_inc = 0;
4300 }
4301 else
4302 {
4303 func(from - prev);
4304 }
4305
4306 for (unsigned i = from+1; i <= to; ++i)
4307 {
4308 func(1);
4309 }
4310 prev = to;
4311 pcurr += 2; // jump to the next positive GAP
4312 }
4313 }
4314
4315 /*!
4316 \brief Convert gap block into array of ints corresponding to 1 bits
4317 @ingroup gapfunc
4318 */
4319 template<typename D, typename T>
4320 D gap_convert_to_arr(D* BMRESTRICT dest,
4321 const T* BMRESTRICT buf,
4322 unsigned dest_len,
4323 bool invert = false) BMNOEXCEPT
4324 {
4325 const T* BMRESTRICT pcurr = buf;
4326 const T* pend = pcurr + (*pcurr >> 3);
4327
4328 D* BMRESTRICT dest_curr = dest;
4329 ++pcurr;
4330
4331 int bitval = (*buf) & 1;
4332 if (invert)
4333 bitval = !bitval; // invert the GAP buffer
4334
4335 if (bitval)
4336 {
4337 if (unsigned(*pcurr + 1) >= dest_len)
4338 return 0; // insufficient space
4339 dest_len -= *pcurr;
4340 T to = *pcurr;
4341 for (T i = 0; ;++i)
4342 {
4343 *dest_curr++ = i;
4344 if (i == to) break;
4345 }
4346 ++pcurr;
4347 }
4348 ++pcurr; // set GAP to 1
4349
4350 while (pcurr <= pend)
4351 {
4352 unsigned pending = *pcurr - *(pcurr-1);
4353 if (pending >= dest_len)
4354 return 0;
4355 dest_len -= pending;
4356 T from = (T)(*(pcurr-1)+1);
4357 T to = *pcurr;
4358 for (T i = from; ;++i)
4359 {
4360 *dest_curr++ = i;
4361 if (i == to) break;
4362 }
4363 pcurr += 2; // jump to the next positive GAP
4364 }
4365 return (D) (dest_curr - dest);
4366 }
4367
4368
4369
4370 /*!
4371 @brief Bitcount for bit block
4372
4373 Function calculates number of 1 bits in the given array of words.
4374 Make sure the addresses are aligned.
4375
4376 @ingroup bitfunc
4377 */
4378 inline
4379 bm::id_t bit_block_count(const bm::word_t* block) BMNOEXCEPT
4380 {
4381 const bm::word_t* block_end = block + bm::set_block_size;
4382 bm::id_t count = 0;
4383
4384 #ifdef BMVECTOPT
4385 count = VECT_BITCOUNT(block, block_end);
4386 #else
4387 #ifdef BM64OPT
4388 // 64-bit optimized algorithm. No sparse vect opt.
4389 // instead it uses 4-way parallel pipelined version
4390
4391 const bm::id64_t* b1 = (bm::id64_t*) block;
4392 const bm::id64_t* b2 = (bm::id64_t*) block_end;
4393 do
4394 {
4395 bm::id64_t x = b1[0];
4396 bm::id64_t y = b1[1];
4397 bm::id64_t u = b1[2];
4398 bm::id64_t v = b1[3];
4399
4400 if (x | y | u | v)
4401 {
4402 unsigned c = bitcount64_4way(x, y, u, v);
4403 BM_ASSERT(c);
4404 count += c;
4405 }
4406 b1 += 4;
4407 } while (b1 < b2);
4408 #else
4409 // For 32 bit code the fastest method is
4410 // to use bitcount table for each byte in the block.
4411 // As optimization for sparse bitsets used bits accumulator
4412 // to collect ON bits using bitwise OR.
4413 bm::word_t acc = *block++;
4414 do
4415 {
4416 bm::word_t in = *block++;
4417 bm::word_t acc_prev = acc;
4418 acc |= in;
4419 if (acc_prev &= in) // accumulator miss: counting bits
4420 {
4421 BM_INCWORD_BITCOUNT(count, acc);
4422 acc = acc_prev;
4423 }
4424 } while (block < block_end);
4425
4426 BM_INCWORD_BITCOUNT(count, acc); // count-in remaining accumulator
4427
4428 #endif
4429 #endif
4430 return count;
4431 }
4432
4433 /*!
4434 @brief Bitcount for bit block
4435
4436 Function calculates number of 1 bits in the given array of words.
4437 uses digest to understand zero areas
4438
4439 @ingroup bitfunc
4440 */
4441 inline
4442 bm::id_t bit_block_count(const bm::word_t* const block,
4443 bm::id64_t digest) BMNOEXCEPT
4444 {
4445 bm::id_t count = 0;
4446 bm::id64_t d = digest;
4447 while (d)
4448 {
4449 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
4450
4451 unsigned wave = bm::word_bitcount64(t - 1);
4452 unsigned off = wave * bm::set_block_digest_wave_size;
4453
4454 const bm::bit_block_t::bunion_t* BMRESTRICT src_u =
4455 (const bm::bit_block_t::bunion_t*)(&block[off]);
4456 unsigned j = 0;
4457 do
4458 {
4459 count += bm::word_bitcount64(src_u->w64[j+0]) +
4460 bm::word_bitcount64(src_u->w64[j+1]) +
4461 bm::word_bitcount64(src_u->w64[j+2]) +
4462 bm::word_bitcount64(src_u->w64[j+3]);
4463 j += 4;
4464 } while (j < bm::set_block_digest_wave_size/2);
4465
4466 d = bm::bmi_bslr_u64(d); // d &= d - 1;
4467 } // while
4468 return count;
4469 }
4470
4471
4472
4473 /*!
4474 @brief Bitcount for bit string
4475
4476 Added for back-compatibility purposes, not block aligned,
4477 not SIMD accelerated
4478
4479 @ingroup bitfunc
4480 */
4481 inline
4482 bm::id_t bit_block_calc_count(const bm::word_t* block,
4483 const bm::word_t* block_end) BMNOEXCEPT
4484 {
4485 bm::id_t count = 0;
4486 bm::word_t acc = *block++;
4487 do
4488 {
4489 bm::word_t in = *block++;
4490 bm::word_t acc_prev = acc;
4491 acc |= in;
4492 if (acc_prev &= in) // accumulator miss: counting bits
4493 {
4494 BM_INCWORD_BITCOUNT(count, acc);
4495 acc = acc_prev;
4496 }
4497 } while (block < block_end);
4498
4499 BM_INCWORD_BITCOUNT(count, acc); // count-in remaining accumulator
4500 return count;
4501 }
4502
4503
4504
4505 /*!
4506 Function calculates number of times when bit value changed
4507 (1-0 or 0-1).
4508
4509 For 001 result is 2
4510 010 - 3
4511 011 - 2
4512 111 - 1
4513
4514 @ingroup bitfunc
4515 */
4516 inline
4517 bm::id_t bit_count_change(bm::word_t w) BMNOEXCEPT
4518 {
4519 unsigned count = 1;
4520 w ^= (w >> 1);
4521
4522 count += bm::word_bitcount(w);
4523 count -= (w >> ((sizeof(w) * 8) - 1));
4524 return count;
4525 }
4526
4527
4528 /*!
4529 Function calculates number of times when bit value changed
4530 @internal
4531 */
4532 inline
4533 unsigned bit_block_change32(const bm::word_t* block, unsigned size) BMNOEXCEPT
4534 {
4535 unsigned gap_count = 1;
4536
4537 bm::word_t w, w0, w_prev, w_l;
4538 w = w0 = *block;
4539
4540 const int w_shift = int(sizeof(w) * 8 - 1);
4541 w ^= (w >> 1);
4542 BM_INCWORD_BITCOUNT(gap_count, w);
4543 gap_count -= (w_prev = (w0 >> w_shift)); // negative value correction
4544
4545 const bm::word_t* block_end = block + size; // bm::set_block_size;
4546 for (++block; block < block_end; ++block)
4547 {
4548 w = w0 = *block;
4549 ++gap_count;
4550 if (!w)
4551 {
4552 gap_count -= !w_prev;
4553 w_prev = 0;
4554 }
4555 else
4556 {
4557 w ^= (w >> 1);
4558 BM_INCWORD_BITCOUNT(gap_count, w);
4559
4560 w_l = w0 & 1;
4561 gap_count -= (w0 >> w_shift); // negative value correction
4562 gap_count -= !(w_prev ^ w_l); // word border correction
4563
4564 w_prev = (w0 >> w_shift);
4565 }
4566 } // for
4567 return gap_count;
4568 }
4569
4570
4571 /*!
4572 Function calculates basic bit-block statistics
4573 number of times when bit value changed (GAPS)
4574 and population count
4575 @param block - bit-block pointer
4576 @param gc - [output] gap_count
4577 @param bc - [output] bit count
4578 @internal
4579 */
4580 inline
4581 void bit_block_change_bc(const bm::word_t* BMRESTRICT block,
4582 unsigned* BMRESTRICT gc, unsigned* BMRESTRICT bc) BMNOEXCEPT
4583 {
4584 BM_ASSERT(gc);
4585 BM_ASSERT(bc);
4586
4587 #ifdef VECT_BLOCK_CHANGE_BC
4588 VECT_BLOCK_CHANGE_BC(block, gc, bc);
4589 #else
4590 *gc = bm::bit_block_change32(block, bm::set_block_size);
4591 *bc = bm::bit_block_count(block);
4592 #endif
4593 }
4594
4595
4596
4597 /*!
4598 Function calculates number of times when bit value changed
4599 (1-0 or 0-1) in the bit block.
4600
4601 @param block - bit-block start pointer
4602 @return number of 1-0, 0-1 transitions
4603
4604 @ingroup bitfunc
4605 */
4606 inline
4607 unsigned bit_block_calc_change(const bm::word_t* block) BMNOEXCEPT
4608 {
4609 #if defined(VECT_BLOCK_CHANGE)
4610 return VECT_BLOCK_CHANGE(block, bm::set_block_size);
4611 #else
4612 return bm::bit_block_change32(block, bm::set_block_size);
4613 #endif
4614 }
4615
4616 /*!
4617 Check if all bits are 1 in [left, right] range
4618 @ingroup bitfunc
4619 */
4620 inline
4621 bool bit_block_is_all_one_range(const bm::word_t* const BMRESTRICT block,
4622 bm::word_t left,
4623 bm::word_t right) BMNOEXCEPT
4624 {
4625 BM_ASSERT(left <= right);
4626 BM_ASSERT(right <= bm::gap_max_bits-1);
4627
4628 unsigned nword, nbit, bitcount, temp;
4629 nbit = left & bm::set_word_mask;
4630 const bm::word_t* word =
4631 block + (nword = unsigned(left >> bm::set_word_shift));
4632 if (left == right) // special case (only 1 bit to check)
4633 return (*word >> nbit) & 1u;
4634
4635 if (nbit) // starting position is not aligned
4636 {
4637 unsigned right_margin = nbit + right - left;
4638 if (right_margin < 32)
4639 {
4640 unsigned mask =
4641 block_set_table<true>::_right[nbit] &
4642 block_set_table<true>::_left[right_margin];
4643 return mask == (*word & mask);
4644 }
4645 temp = *word & block_set_table<true>::_right[nbit];
4646 if (temp != block_set_table<true>::_right[nbit])
4647 return false;
4648 bitcount = (right - left + 1u) - (32 - nbit);
4649 ++word;
4650 }
4651 else
4652 {
4653 bitcount = right - left + 1u;
4654 }
4655
4656 // now when we are word aligned, we can scan the bit-stream
4657 const bm::id64_t maskFF64 = ~0ull;
4658 const bm::word_t maskFF = ~0u;
4659 // loop unrolled to evaluate 4 words at a time
4660 // SIMD showed no advantage, unless evaluate sub-wave intervals
4661 //
4662 for ( ;bitcount >= 128; bitcount-=128, word+=4)
4663 {
4664 bm::id64_t w64_0 = bm::id64_t(word[0]) + (bm::id64_t(word[1]) << 32);
4665 bm::id64_t w64_1 = bm::id64_t(word[2]) + (bm::id64_t(word[3]) << 32);
4666 if ((w64_0 ^ maskFF64) | (w64_1 ^ maskFF64))
4667 return false;
4668 } // for
4669
4670 for ( ;bitcount >= 32; bitcount-=32, ++word)
4671 {
4672 if (*word != maskFF)
4673 return false;
4674 } // for
4675 BM_ASSERT(bitcount < 32);
4676
4677 if (bitcount) // we have a tail to count
4678 {
4679 temp = *word & block_set_table<true>::_left[bitcount-1];
4680 if (temp != block_set_table<true>::_left[bitcount-1])
4681 return false;
4682 }
4683
4684 return true;
4685 }
4686
4687
4688
4689
4690 /*!
4691 Function calculates number of 1 bits in the given array of words in
4692 the range between left anf right bits (borders included)
4693 Make sure the addr is aligned.
4694
4695 @ingroup bitfunc
4696 */
4697 inline
4698 bm::id_t bit_block_calc_count_range(const bm::word_t* block,
4699 bm::word_t left,
4700 bm::word_t right) BMNOEXCEPT
4701 {
4702 BM_ASSERT(left <= right);
4703 BM_ASSERT(right <= bm::gap_max_bits-1);
4704
4705 unsigned nword, nbit, bitcount, count;
4706 nbit = left & bm::set_word_mask;
4707 const bm::word_t* word =
4708 block + (nword = unsigned(left >> bm::set_word_shift));
4709 if (left == right) // special case (only 1 bit to check)
4710 {
4711 return (*word >> nbit) & 1u;
4712 }
4713
4714 count = 0;
4715 bitcount = right - left + 1u;
4716 if (nbit) // starting position is not aligned
4717 {
4718 unsigned right_margin = nbit + right - left;
4719 if (right_margin < 32)
4720 {
4721 unsigned mask =
4722 block_set_table<true>::_right[nbit] &
4723 block_set_table<true>::_left[right_margin];
4724 return bm::word_bitcount(*word & mask);
4725 }
4726 count = bm::word_bitcount(*word & block_set_table<true>::_right[nbit]);
4727 bitcount -= 32 - nbit;
4728 ++word;
4729 }
4730
4731 // now when we are word aligned, we can count bits the usual way
4732 //
4733 #if defined(BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
4734 for ( ;bitcount >= 64; bitcount-=64)
4735 {
4736 bm::id64_t w64 = *((bm::id64_t*)word); // x86 unaligned(!) read
4737 count += unsigned(_mm_popcnt_u64(w64));
4738 word += 2;
4739 }
4740 if (bitcount >= 32)
4741 {
4742 count += bm::word_bitcount(*word++);
4743 bitcount-=32;
4744 }
4745 #else
4746 for ( ;bitcount >= 32; bitcount-=32, ++word)
4747 count += bm::word_bitcount(*word);
4748 #endif
4749 BM_ASSERT(bitcount < 32);
4750
4751 if (bitcount) // we have a tail to count
4752 {
4753 count += bm::word_bitcount(
4754 *word & block_set_table<true>::_left[bitcount-1]);
4755 }
4756 return count;
4757 }
4758
4759 /*!
4760 Function calculates number of 1 bits in the given array of words in
4761 the range between 0 anf right bits (borders included)
4762 Make sure the addr is aligned.
4763
4764 @ingroup bitfunc
4765 */
4766 inline
4767 bm::id_t bit_block_calc_count_to(const bm::word_t* block,
4768 bm::word_t right) BMNOEXCEPT
4769 {
4770 BM_ASSERT(block);
4771 if (!right) // special case, first bit check
4772 return *block & 1u;
4773 bm::id_t count = 0;
4774
4775 unsigned bitcount = right + 1;
4776
4777 // AVX2 or 64-bit loop unroll
4778 #if defined(BMAVX2OPT) || defined(BMAVX512OPT)
4779 BM_AVX2_POPCNT_PROLOG
4780
4781 __m256i cnt = _mm256_setzero_si256();
4782 bm::id64_t* cnt64;
4783
4784 for ( ;bitcount >= 256; bitcount -= 256)
4785 {
4786 const __m256i* src = (__m256i*)block;
4787 __m256i xmm256 = _mm256_load_si256(src);
4788 BM_AVX2_BIT_COUNT(bc, xmm256);
4789 cnt = _mm256_add_epi64(cnt, bc);
4790
4791 block += 8;
4792 }
4793 cnt64 = (bm::id64_t*)&cnt;
4794 count += (unsigned)(cnt64[0] + cnt64[1] + cnt64[2] + cnt64[3]);
4795 #endif
4796
4797 for ( ;bitcount >= 64; bitcount -= 64)
4798 {
4799 bm::id64_t* p = (bm::id64_t*)block;
4800 count += bm::word_bitcount64(*p);
4801 block += 2;
4802 }
4803 if (bitcount >= 32)
4804 {
4805 count += bm::word_bitcount(*block++);
4806 bitcount-=32;
4807 }
4808
4809 if (bitcount) // we have a tail to count
4810 {
4811 count +=
4812 bm::word_bitcount(*block & block_set_table<true>::_left[bitcount-1]);
4813 }
4814 return count;
4815 }
4816
4817
4818
4819 /*!
4820 Cyclic rotation of bit-block left by 1 bit
4821 @ingroup bitfunc
4822 */
4823 inline
4824 void bit_block_rotate_left_1(bm::word_t* block) BMNOEXCEPT
4825 {
4826 bm::word_t co_flag = (block[0] >> 31) & 1; // carry over bit
4827 for (unsigned i = 0; i < bm::set_block_size-1; ++i)
4828 {
4829 block[i] = (block[i] << 1) | (block[i + 1] >> 31);
4830 }
4831 block[set_block_size - 1] = (block[set_block_size - 1] << 1) | co_flag;
4832 }
4833
4834 /*!
4835 @brief Unrolled cyclic rotation of bit-block left by 1 bit
4836 @param block - bit-block pointer
4837 @ingroup bitfunc
4838 */
4839 inline
4840 void bit_block_rotate_left_1_unr(bm::word_t* block) BMNOEXCEPT
4841 {
4842 bm::word_t co_flag = (block[0] >> 31) & 1; // carry over bit
4843 const unsigned unroll_factor = 4;
4844 bm::word_t w0, w1, w2, w3;
4845
4846 unsigned i;
4847 for (i = 0; i < bm::set_block_size - unroll_factor; i += unroll_factor)
4848 {
4849 w0 = block[i + 1] >> 31;
4850 w1 = block[i + 2] >> 31;
4851 w2 = block[i + 3] >> 31;
4852 w3 = block[i + 4] >> 31;
4853
4854 block[0 + i] = (block[0 + i] << 1) | w0;
4855 block[1 + i] = (block[1 + i] << 1) | w1;
4856 block[2 + i] = (block[2 + i] << 1) | w2;
4857 block[3 + i] = (block[3 + i] << 1) | w3;
4858 }
4859 block[i] = (block[i] << 1) | (block[i + 1] >> 31);
4860 block[i + 1] = (block[i + 1] << 1) | (block[i + 2] >> 31);
4861 block[i + 2] = (block[i + 2] << 1) | (block[i + 3] >> 31);
4862 block[set_block_size - 1] = (block[set_block_size - 1] << 1) | co_flag;
4863 }
4864
4865 /*!
4866 @brief insert bit into position and shift the rest right with carryover
4867
4868 @param block - bit-block pointer
4869 @param bitpos - bit position to insert
4870 @param value - bit value (0|1) to insert
4871
4872 @return carry over value
4873 @ingroup bitfunc
4874 */
4875 inline
4876 bm::word_t bit_block_insert(bm::word_t* BMRESTRICT block,
4877 unsigned bitpos, bool value) BMNOEXCEPT
4878 {
4879 BM_ASSERT(block);
4880 BM_ASSERT(bitpos < 65536);
4881
4882 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
4883 unsigned nword = unsigned(nbit >> bm::set_word_shift);
4884 nbit &= bm::set_word_mask;
4885
4886 bm::word_t co_flag = 0;
4887 if (nbit)
4888 {
4889 unsigned r_mask = bm::block_set_table<true>::_right[nbit];
4890 bm::word_t w = block[nword] & r_mask;
4891 bm::word_t wl = block[nword] & ~r_mask;
4892 co_flag = w >> 31;
4893 w <<= 1u;
4894 block[nword] = w | (unsigned(value) << nbit) | wl;
4895 ++nword;
4896 }
4897 else
4898 {
4899 co_flag = value;
4900 }
4901
4902 for (unsigned i = nword; i < bm::set_block_size; ++i)
4903 {
4904 bm::word_t w = block[i];
4905 bm::word_t co_flag1 = w >> 31;
4906 w = (w << 1u) | co_flag;
4907 block[i] = w;
4908 co_flag = co_flag1;
4909 } // for i
4910 return co_flag;
4911 }
4912
4913
4914
4915 /*!
4916 @brief Right bit-shift bitblock by 1 bit (reference)
4917 @param block - bit-block pointer
4918 @param empty_acc - [out] contains 0 if block is empty
4919 @param co_flag - carry over from the previous block
4920
4921 @return carry over bit (1 or 0)
4922 @ingroup bitfunc
4923 */
4924 inline
4925 bool bit_block_shift_r1(bm::word_t* BMRESTRICT block,
4926 bm::word_t* BMRESTRICT empty_acc,
4927 bm::word_t co_flag) BMNOEXCEPT
4928 {
4929 BM_ASSERT(block);
4930 BM_ASSERT(empty_acc);
4931
4932 bm::word_t acc = 0;
4933 for (unsigned i = 0; i < bm::set_block_size; ++i)
4934 {
4935 bm::word_t w = block[i];
4936 bm::word_t co_flag1 = w >> 31;
4937 acc |= w = (w << 1u) | co_flag;
4938 block[i] = w;
4939 co_flag = co_flag1;
4940 }
4941 *empty_acc = acc;
4942 return co_flag;
4943 }
4944
4945 /*!
4946 @brief Right bit-shift of bit-block by 1 bit (loop unrolled)
4947 @param block - bit-block pointer
4948 @param empty_acc - [out] contains 0 if block is empty
4949 @param co_flag - carry over from the previous block
4950
4951 @return carry over bit (1 or 0)
4952 @ingroup bitfunc
4953 */
4954 inline
4955 bool bit_block_shift_r1_unr(bm::word_t* BMRESTRICT block,
4956 bm::word_t* BMRESTRICT empty_acc,
4957 bm::word_t co_flag) BMNOEXCEPT
4958 {
4959 BM_ASSERT(block);
4960 BM_ASSERT(empty_acc);
4961 #if defined(VECT_SHIFT_R1)
4962 return VECT_SHIFT_R1(block, empty_acc, co_flag);
4963 #else
4964 return bm::bit_block_shift_r1(block, empty_acc, co_flag);
4965 #endif
4966 }
4967
4968
4969 /*!
4970 @brief Left bit-shift bitblock by 1 bit (reference)
4971 @param block - bit-block pointer
4972 @param empty_acc - [out] contains 0 if block is empty
4973 @param co_flag - carry over from the prev/next block
4974
4975 @return carry over bit (1 or 0)
4976
4977 @ingroup bitfunc
4978 */
4979 inline
4980 bool bit_block_shift_l1(bm::word_t* block,
4981 bm::word_t* empty_acc, bm::word_t co_flag) BMNOEXCEPT
4982 {
4983 BM_ASSERT(block);
4984 BM_ASSERT(empty_acc);
4985
4986 bm::word_t acc = 0;
4987 for (int i = bm::set_block_size-1; i >= 0; --i)
4988 {
4989 bm::word_t w = block[i];
4990 bm::word_t co_flag1 = w & 1u;
4991 acc |= w = (w >> 1u) | (co_flag << 31u);
4992 block[i] = w;
4993 co_flag = co_flag1;
4994 }
4995
4996 *empty_acc = acc;
4997 return co_flag;
4998 }
4999
5000 /*!
5001 @brief Left bit-shift of bit-block by 1 bit (loop unrolled)
5002 @param block - bit-block pointer
5003 @param empty_acc - [out] contains 0 if block is empty
5004 @param co_flag - carry over from the prev/next block
5005
5006 @return carry over bit (1 or 0)
5007 @ingroup bitfunc
5008 */
5009 inline
5010 bool bit_block_shift_l1_unr(bm::word_t* block,
5011 bm::word_t* empty_acc,
5012 bm::word_t co_flag) BMNOEXCEPT
5013 {
5014 BM_ASSERT(block);
5015 BM_ASSERT(empty_acc);
5016 #if defined(VECT_SHIFT_L1)
5017 return VECT_SHIFT_L1(block, empty_acc, co_flag);
5018 #else
5019 return bm::bit_block_shift_l1(block, empty_acc, co_flag);
5020 #endif
5021 }
5022
5023 /*!
5024 @brief erase bit from position and shift the rest right with carryover
5025
5026 @param block - bit-block pointer
5027 @param bitpos - bit position to insert
5028 @param carry_over - bit value to add to the end (0|1)
5029
5030 @ingroup bitfunc
5031 */
5032 inline
5033 void bit_block_erase(bm::word_t* block,
5034 unsigned bitpos,
5035 bool carry_over) BMNOEXCEPT
5036 {
5037 BM_ASSERT(block);
5038 BM_ASSERT(bitpos < 65536);
5039
5040 if (!bitpos)
5041 {
5042 bm::word_t acc;
5043 bm::bit_block_shift_l1_unr(block, &acc, carry_over);
5044 return;
5045 }
5046
5047 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
5048 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5049 nbit &= bm::set_word_mask;
5050
5051 bm::word_t co_flag = carry_over;
5052 for (unsigned i = bm::set_block_size-1; i > nword; --i)
5053 {
5054 bm::word_t w = block[i];
5055 bm::word_t co_flag1 = w & 1u;
5056 w = (w >> 1u) | (co_flag << 31u);
5057 block[i] = w;
5058 co_flag = co_flag1;
5059 }
5060
5061 if (nbit)
5062 {
5063 unsigned r_mask = bm::block_set_table<true>::_right[nbit];
5064 bm::word_t w = block[nword] & r_mask;
5065 bm::word_t wl = block[nword] & ~r_mask;
5066 w &= ~(1 << nbit); // clear the removed bit
5067 w >>= 1u;
5068 w |= wl | (co_flag << 31u);
5069 block[nword] = w;
5070 }
5071 else
5072 {
5073 block[nword] = (block[nword] >> 1u) | (co_flag << 31u);
5074 }
5075 }
5076
5077
5078 /*!
5079 @brief Right bit-shift of bit-block by 1 bit (reference) + AND
5080 @param block - bit-block pointer
5081 @param co_flag - carry over from the previous block
5082 @param mask_block - mask bit-block pointer
5083 @param digest - block digest
5084
5085 @return carry over bit (1 or 0)
5086 @ingroup bitfunc
5087 */
5088 inline
5089 bool bit_block_shift_r1_and(bm::word_t* BMRESTRICT block,
5090 bm::word_t co_flag,
5091 const bm::word_t* BMRESTRICT mask_block,
5092 bm::id64_t* BMRESTRICT digest) BMNOEXCEPT
5093 {
5094 BM_ASSERT(block);
5095 BM_ASSERT(mask_block);
5096 BM_ASSERT(digest && *digest);
5097
5098
5099 bm::id64_t d = *digest;
5100
5101 unsigned di = 0;
5102 if (!co_flag)
5103 {
5104 bm::id64_t t = d & -d;
5105 di = bm::word_bitcount64(t - 1); // find start bit-index
5106 }
5107
5108 for (; di < 64; ++di)
5109 {
5110 const unsigned d_base = di * bm::set_block_digest_wave_size;
5111 bm::id64_t dmask = (1ull << di);
5112 if (d & dmask) // digest stride not empty
5113 {
5114 bm::word_t acc = 0;
5115 for (unsigned i = d_base; i < d_base + bm::set_block_digest_wave_size; ++i)
5116 {
5117 BM_ASSERT(i < bm::set_block_size);
5118
5119 bm::word_t w = block[i];
5120 bm::word_t co_flag1 = w >> 31;
5121 w = (w << 1u) | co_flag;
5122 acc |= block[i] = w & mask_block[i];
5123 co_flag = co_flag1;
5124 }
5125 if (!acc)
5126 d &= ~dmask; // update digest: clear stride bit
5127 }
5128 else // stride is empty
5129 {
5130 BM_ASSERT(block[d_base + bm::set_block_digest_wave_size -1]==0);
5131 BM_ASSERT(block[d_base]==0);
5132
5133 if (co_flag) // there is carry-over
5134 {
5135 BM_ASSERT(co_flag == 1);
5136 BM_ASSERT(block[d_base] == 0);
5137
5138 block[d_base] = co_flag & mask_block[d_base];
5139 if (block[d_base])
5140 d |= dmask; // update digest
5141 co_flag = 0;
5142 }
5143 }
5144 } // for di
5145
5146 *digest = d;
5147 return co_flag;
5148 }
5149
5150 /*!
5151 @brief Right bit-shift bitblock by 1 bit (reference) + AND
5152 @param block - bit-block pointer
5153 @param co_flag - carry over from the previous block
5154 @param mask_block - mask bit-block pointer
5155 @param digest - block digest
5156
5157 @return carry over bit (1 or 0)
5158 @ingroup bitfunc
5159 */
5160 inline
5161 bool bit_block_shift_r1_and_unr(bm::word_t* BMRESTRICT block,
5162 bm::word_t co_flag,
5163 const bm::word_t* BMRESTRICT mask_block,
5164 bm::id64_t* BMRESTRICT digest) BMNOEXCEPT
5165 {
5166 BM_ASSERT(block);
5167 BM_ASSERT(mask_block);
5168 BM_ASSERT(digest);
5169
5170 #if defined(VECT_SHIFT_R1_AND)
5171 return VECT_SHIFT_R1_AND(block, co_flag, mask_block, digest);
5172 #else
5173 return bm::bit_block_shift_r1_and(block, co_flag, mask_block, digest);
5174 #endif
5175 }
5176
5177
5178 /*!
5179 Function calculates if there is any number of 1 bits
5180 in the given array of words in the range between left anf right bits
5181 (borders included). Make sure the addresses are aligned.
5182
5183 @ingroup bitfunc
5184 */
5185 inline
5186 bm::id_t bit_block_any_range(const bm::word_t* const BMRESTRICT block,
5187 bm::word_t left,
5188 bm::word_t right) BMNOEXCEPT
5189 {
5190 BM_ASSERT(left <= right);
5191
5192 unsigned nbit = left; // unsigned(left & bm::set_block_mask);
5193 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5194 nbit &= bm::set_word_mask;
5195
5196 const bm::word_t* word = block + nword;
5197
5198 if (left == right) // special case (only 1 bit to check)
5199 {
5200 return (*word >> nbit) & 1;
5201 }
5202 unsigned acc;
5203 unsigned bitcount = right - left + 1;
5204
5205 if (nbit) // starting position is not aligned
5206 {
5207 unsigned right_margin = nbit + (right - left);
5208 if (right_margin < 32)
5209 {
5210 unsigned mask =
5211 block_set_table<true>::_right[nbit] &
5212 block_set_table<true>::_left[right_margin];
5213 return *word & mask;
5214 }
5215 else
5216 {
5217 acc = *word & block_set_table<true>::_right[nbit];
5218 if (acc)
5219 return acc;
5220 bitcount -= 32 - nbit;
5221 }
5222 ++word;
5223 }
5224
5225 // loop unrolled to evaluate 4 words at a time
5226 // SIMD showed no advantage, unless evaluate sub-wave intervals
5227 //
5228 for ( ;bitcount >= 128; bitcount-=128, word+=4)
5229 {
5230 acc = word[0] | word[1] | word[2] | word[3];
5231 if (acc)
5232 return acc;
5233 } // for
5234
5235 acc = 0;
5236 for ( ;bitcount >= 32; bitcount -= 32)
5237 {
5238 acc |= *word++;
5239 } // for
5240
5241 if (bitcount) // we have a tail to count
5242 acc |= (*word) & block_set_table<true>::_left[bitcount-1];
5243
5244 return acc;
5245 }
5246
5247 // ----------------------------------------------------------------------
5248
5249 /*! Function inverts block of bits
5250 @ingroup bitfunc
5251 */
5252 template<typename T>
5253 void bit_invert(T* start) BMNOEXCEPT
5254 {
5255 BM_ASSERT(IS_VALID_ADDR((bm::word_t*)start));
5256 #ifdef BMVECTOPT
5257 VECT_INVERT_BLOCK(start);
5258 #else
5259 T* end = (T*)((unsigned*)(start) + bm::set_block_size);
5260 do
5261 {
5262 start[0] = ~start[0];
5263 start[1] = ~start[1];
5264 start[2] = ~start[2];
5265 start[3] = ~start[3];
5266 start+=4;
5267 } while (start < end);
5268 #endif
5269 }
5270
5271 // ----------------------------------------------------------------------
5272
5273 /*! @brief Returns "true" if all bits in the block are 1
5274 @ingroup bitfunc
5275 */
5276 inline
5277 bool is_bits_one(const bm::wordop_t* start) BMNOEXCEPT
5278 {
5279 #if defined(BMSSE42OPT) || defined(BMAVX2OPT)
5280 return VECT_IS_ONE_BLOCK(start);
5281 #else
5282 const bm::word_t* BMRESTRICT src_end = (bm::word_t*)start + bm::set_block_size;
5283 const bm::wordop_t* end = (const bm::wordop_t*)src_end;
5284 do
5285 {
5286 bm::wordop_t tmp =
5287 start[0] & start[1] & start[2] & start[3];
5288 if (tmp != bm::all_bits_mask)
5289 return false;
5290 start += 4;
5291 } while (start < end);
5292 return true;
5293 #endif
5294 }
5295
5296 // ----------------------------------------------------------------------
5297
5298 /*! @brief Returns "true" if all bits are 1 in the block [left, right]
5299 Function check for block varieties
5300 @internal
5301 */
5302 inline
5303 bool block_is_all_one_range(const bm::word_t* const BMRESTRICT block,
5304 unsigned left, unsigned right) BMNOEXCEPT
5305 {
5306 BM_ASSERT(left <= right);
5307 BM_ASSERT(right < bm::gap_max_bits);
5308 if (block)
5309 {
5310 if (BM_IS_GAP(block))
5311 return bm::gap_is_all_one_range(BMGAP_PTR(block), left, right);
5312 if (block == FULL_BLOCK_FAKE_ADDR)
5313 return true;
5314 return bm::bit_block_is_all_one_range(block, left, right);
5315 }
5316 return false;
5317 }
5318
5319 /*! @brief Returns "true" if all bits are 1 in the block [left, right]
5320 and border bits are 0
5321 @internal
5322 */
5323 inline
5324 bool block_is_interval(const bm::word_t* const BMRESTRICT block,
5325 unsigned left, unsigned right) BMNOEXCEPT
5326 {
5327 BM_ASSERT(left <= right);
5328 BM_ASSERT(right < bm::gap_max_bits-1);
5329
5330 if (block)
5331 {
5332 bool is_left, is_right, all_one;
5333 if (BM_IS_GAP(block))
5334 {
5335 const bm::gap_word_t* gap = BMGAP_PTR(block);
5336 all_one = bm::gap_is_interval(gap, left, right);
5337 return all_one;
5338 }
5339 else // bit-block
5340 {
5341 if (block == FULL_BLOCK_FAKE_ADDR)
5342 return false;
5343 unsigned nword = ((left-1) >> bm::set_word_shift);
5344 is_left = block[nword] & (1u << ((left-1) & bm::set_word_mask));
5345 if (is_left == false)
5346 {
5347 nword = ((right + 1) >> bm::set_word_shift);
5348 is_right = block[nword] & (1u << ((right + 1) & bm::set_word_mask));
5349 if (is_right == false)
5350 {
5351 all_one = bm::bit_block_is_all_one_range(block, left, right);
5352 return all_one;
5353 }
5354 }
5355 }
5356 }
5357
5358 return false;
5359 }
5360
5361 // ----------------------------------------------------------------------
5362
5363 /**
5364 \brief Searches for the last 1 bit in the 111 interval of a BIT block
5365 \param block - BIT buffer
5366 \param nbit - bit index to start checking from
5367 \param pos - [out] found value
5368
5369 \return false if not found
5370 @ingroup bitfunc
5371 */
5372 inline
5373 bool bit_block_find_interval_end(const bm::word_t* BMRESTRICT block,
5374 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
5375 {
5376 BM_ASSERT(block);
5377 BM_ASSERT(pos);
5378
5379 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5380 unsigned bit_pos = (nbit & bm::set_word_mask);
5381 bm::word_t w = block[nword];
5382 w &= (1u << bit_pos);
5383 if (!w)
5384 return false;
5385
5386 if (nbit == bm::gap_max_bits-1)
5387 {
5388 *pos = bm::gap_max_bits-1;
5389 return true;
5390 }
5391 *pos = nbit;
5392
5393 ++nbit;
5394 nword = unsigned(nbit >> bm::set_word_shift);
5395 bit_pos = (nbit & bm::set_word_mask);
5396
5397 w = (~block[nword]) >> bit_pos;
5398 w <<= bit_pos; // clear the trailing bits
5399 if (w)
5400 {
5401 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
5402 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))-1);
5403 return true;
5404 }
5405
5406 for (++nword; nword < bm::set_block_size; ++nword)
5407 {
5408 w = ~block[nword];
5409 if (w)
5410 {
5411 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
5412 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))-1);
5413 return true;
5414 }
5415 } // for nword
5416
5417 // 0 not found, all block is 1s...
5418 *pos = bm::gap_max_bits-1;
5419 return true;
5420 }
5421
5422
5423 /*! @brief Find end of the current 111 interval
5424 @return search result code 0 - not found, 1 found, 2 - found at the end
5425 @internal
5426 */
5427 inline
5428 unsigned block_find_interval_end(const bm::word_t* BMRESTRICT block,
5429 unsigned nbit_from,
5430 unsigned* BMRESTRICT found_nbit) BMNOEXCEPT
5431 {
5432 BM_ASSERT(block && found_nbit);
5433 BM_ASSERT(nbit_from < bm::gap_max_bits);
5434
5435 bool b;
5436 if (BM_IS_GAP(block))
5437 {
5438 const bm::gap_word_t* gap = BMGAP_PTR(block);
5439 b = bm::gap_find_interval_end(gap, nbit_from, found_nbit);
5440 if (b && *found_nbit == bm::gap_max_bits-1)
5441 return 2; // end of block, keep searching
5442 }
5443 else // bit-block
5444 {
5445 if (IS_FULL_BLOCK(block))
5446 {
5447 *found_nbit = bm::gap_max_bits-1;
5448 return 2;
5449 }
5450 b = bm::bit_block_find_interval_end(block, nbit_from, found_nbit);
5451 if (b && *found_nbit == bm::gap_max_bits-1)
5452 return 2; // end of block, keep searching
5453 }
5454 return b;
5455 }
5456
5457 // ----------------------------------------------------------------------
5458
5459 /**
5460 \brief Searches for the first 1 bit in the 111 interval of a BIT block
5461 \param block - BIT buffer
5462 \param nbit - bit index to start checking from
5463 \param pos - [out] found value
5464
5465 \return false if not found
5466 @ingroup bitfunc
5467 */
5468 inline
5469 bool bit_block_find_interval_start(const bm::word_t* BMRESTRICT block,
5470 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
5471 {
5472 BM_ASSERT(block);
5473 BM_ASSERT(pos);
5474
5475 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5476 unsigned bit_pos = (nbit & bm::set_word_mask);
5477 bm::word_t w = block[nword];
5478 w &= (1u << bit_pos);
5479 if (!w)
5480 return false;
5481
5482 if (nbit == 0)
5483 {
5484 *pos = 0;
5485 return true;
5486 }
5487 *pos = nbit;
5488
5489 --nbit;
5490 nword = unsigned(nbit >> bm::set_word_shift);
5491 bit_pos = (nbit & bm::set_word_mask);
5492
5493 w = (~block[nword]) & block_set_table<true>::_left[bit_pos];
5494 if (w)
5495 {
5496 bit_pos = bm::bit_scan_reverse32(w);
5497 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))+1);
5498 return true;
5499 }
5500
5501 if (nword)
5502 {
5503 for (--nword; true; --nword)
5504 {
5505 w = ~block[nword];
5506 if (w)
5507 {
5508 bit_pos = bm::bit_scan_reverse32(w); // trailing zeros
5509 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))+1);
5510 return true;
5511 }
5512 if (!nword)
5513 break;
5514 } // for nword
5515 }
5516
5517 // 0 not found, all block is 1s...
5518 *pos = 0;
5519 return true;
5520 }
5521
5522
5523 /*! @brief Find start of the current 111 interval
5524 @return search result code 0 - not found, 1 found, 2 - found at the start
5525 @internal
5526 */
5527 inline
5528 unsigned block_find_interval_start(const bm::word_t* BMRESTRICT block,
5529 unsigned nbit_from,
5530 unsigned* BMRESTRICT found_nbit) BMNOEXCEPT
5531 {
5532 BM_ASSERT(block && found_nbit);
5533 BM_ASSERT(nbit_from < bm::gap_max_bits);
5534 bool b;
5535 if (BM_IS_GAP(block))
5536 {
5537 const bm::gap_word_t* gap = BMGAP_PTR(block);
5538 b = bm::gap_find_interval_start(gap, nbit_from, found_nbit);
5539 if (b && *found_nbit == 0)
5540 return 2; // start of block, keep searching
5541 }
5542 else // bit-block
5543 {
5544 if (IS_FULL_BLOCK(block))
5545 {
5546 *found_nbit = 0;
5547 return 2;
5548 }
5549 b = bm::bit_block_find_interval_start(block, nbit_from, found_nbit);
5550 if (b && *found_nbit == 0)
5551 return 2; // start of block, keep searching
5552 }
5553 return b;
5554 }
5555
5556 // ----------------------------------------------------------------------
5557
5558 /*! @brief Returns "true" if one bit is set in the block [left, right]
5559 Function check for block varieties
5560 @internal
5561 */
5562 inline
5563 bool block_any_range(const bm::word_t* const BMRESTRICT block,
5564 unsigned left, unsigned right) BMNOEXCEPT
5565 {
5566 BM_ASSERT(left <= right);
5567 BM_ASSERT(right < bm::gap_max_bits);
5568 if (!block)
5569 return false;
5570 if (BM_IS_GAP(block))
5571 return bm::gap_any_range(BMGAP_PTR(block), left, right);
5572 if (IS_FULL_BLOCK(block))
5573 return true;
5574 return bm::bit_block_any_range(block, left, right);
5575 }
5576
5577 // ----------------------------------------------------------------------
5578
5579 /*! @brief Returns "true" if one bit is set in the block
5580 Function check for block varieties
5581 @internal
5582 */
5583 inline
5584 bool block_any(const bm::word_t* const BMRESTRICT block) BMNOEXCEPT
5585 {
5586 if (!block)
5587 return false;
5588 if (IS_FULL_BLOCK(block))
5589 return true;
5590 bool all_zero = (BM_IS_GAP(block)) ?
5591 bm::gap_is_all_zero(BMGAP_PTR(block))
5592 : bm::bit_is_all_zero(block);
5593 return !all_zero;
5594 }
5595
5596
5597
5598 // ----------------------------------------------------------------------
5599
5600 // GAP blocks manipulation functions:
5601
5602 /*! \brief GAP and functor */
5603 BMFORCEINLINE unsigned and_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5604 {
5605 return v1 & v2;
5606 }
5607
5608
5609 /*! \brief GAP xor functor */
5610 BMFORCEINLINE unsigned xor_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5611 {
5612 return v1 ^ v2;
5613 }
5614
5615
5616 /*! \brief GAP or functor */
5617 BMFORCEINLINE unsigned or_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5618 {
5619 return v1 | v2;
5620 }
5621
5622 /*! \brief GAP or functor */
5623 BMFORCEINLINE unsigned sub_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5624 {
5625 return v1 & ~v2;
5626 }
5627
5628
5629 /*!
5630 \brief GAP AND operation.
5631
5632 Function performs AND logical operation on gap vectors.
5633 If possible function put the result into vect1 and returns this
5634 pointer. Otherwise result is put into tmp_buf, which should be
5635 twice of the vector size.
5636
5637 \param vect1 - operand 1
5638 \param vect2 - operand 2
5639 \param tmp_buf - pointer on temporary buffer
5640 \param dsize - out size of the destination
5641 \return Result pointer (tmp_buf OR vect1)
5642
5643 @ingroup gapfunc
5644 */
5645 BMFORCEINLINE
5646 gap_word_t* gap_operation_and(const gap_word_t* BMRESTRICT vect1,
5647 const gap_word_t* BMRESTRICT vect2,
5648 gap_word_t* BMRESTRICT tmp_buf,
5649 unsigned& dsize) BMNOEXCEPT
5650 {
5651 bm::gap_buff_op(tmp_buf, vect1, 0, vect2, 0, bm::and_op, dsize);
5652 return tmp_buf;
5653 }
5654
5655 /*!
5656 \brief GAP AND operation test.
5657
5658 Function performs AND logical operation on gap vectors.
5659 If possible function put the result into vect1 and returns this
5660 pointer. Otherwise result is put into tmp_buf, which should be
5661 twice of the vector size.
5662
5663 \param vect1 - operand 1
5664 \param vect2 - operand 2
5665 \return non zero value if operation returns any 1 bit
5666
5667 @ingroup gapfunc
5668 */
5669 BMFORCEINLINE
5670 unsigned gap_operation_any_and(const gap_word_t* BMRESTRICT vect1,
5671 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5672 {
5673 return gap_buff_any_op(vect1, 0, vect2, 0, and_op);
5674 }
5675
5676
5677 /*!
5678 \brief GAP bitcount AND operation test.
5679
5680 \param vect1 - operand 1
5681 \param vect2 - operand 2
5682 \return bitcount of vect1 AND vect2
5683
5684 @ingroup gapfunc
5685 */
5686 inline
5687 unsigned gap_count_and(const gap_word_t* BMRESTRICT vect1,
5688 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5689 {
5690 return gap_buff_count_op(vect1, vect2, and_op);
5691 }
5692
5693
5694
5695 /*!
5696 \brief GAP XOR operation.
5697
5698 Function performs XOR logical operation on gap vectors.
5699 If possible function put the result into vect1 and returns this
5700 pointer. Otherwise result is put into tmp_buf, which should be
5701 twice of the vector size.
5702
5703 \param vect1 - operand 1
5704 \param vect2 - operand 2
5705 \param tmp_buf - pointer on temporary buffer
5706 \param dsize - out destination size
5707 \return Result pointer (tmp_buf)
5708
5709 @ingroup gapfunc
5710 */
5711 inline
5712 gap_word_t* gap_operation_xor(const gap_word_t* BMRESTRICT vect1,
5713 const gap_word_t* BMRESTRICT vect2,
5714 gap_word_t* BMRESTRICT tmp_buf,
5715 unsigned& dsize) BMNOEXCEPT
5716 {
5717 gap_buff_op(tmp_buf, vect1, 0, vect2, 0, bm::xor_op, dsize);
5718 return tmp_buf;
5719 }
5720
5721 /*! Light weight gap_operation_xor for len prediction
5722 @ingroup gapfunc
5723 */
5724 inline
5725 bool gap_operation_dry_xor(const gap_word_t* BMRESTRICT vect1,
5726 const gap_word_t* BMRESTRICT vect2,
5727 unsigned& dsize,
5728 unsigned limit) BMNOEXCEPT
5729 {
5730 return gap_buff_dry_op(vect1, vect2, bm::xor_op, dsize, limit);
5731 }
5732
5733
5734 /*!
5735 \brief GAP XOR operation test.
5736
5737 Function performs AND logical operation on gap vectors.
5738 If possible function put the result into vect1 and returns this
5739 pointer. Otherwise result is put into tmp_buf, which should be
5740 twice of the vector size.
5741
5742 \param vect1 - operand 1
5743 \param vect2 - operand 2
5744 \return non zero value if operation returns any 1 bit
5745
5746 @ingroup gapfunc
5747 */
5748 BMFORCEINLINE
5749 unsigned gap_operation_any_xor(const gap_word_t* BMRESTRICT vect1,
5750 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5751 {
5752 return gap_buff_any_op(vect1, 0, vect2, 0, bm::xor_op);
5753 }
5754
5755 /*!
5756 \brief GAP bitcount XOR operation test.
5757
5758 \param vect1 - operand 1
5759 \param vect2 - operand 2
5760 \return bitcount of vect1 XOR vect2
5761
5762 @ingroup gapfunc
5763 */
5764 BMFORCEINLINE
5765 unsigned gap_count_xor(const gap_word_t* BMRESTRICT vect1,
5766 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5767 {
5768 return gap_buff_count_op(vect1, vect2, bm::xor_op);
5769 }
5770
5771
5772 /*!
5773 \brief GAP OR operation.
5774
5775 Function performs OR logical oparation on gap vectors.
5776 If possible function put the result into vect1 and returns this
5777 pointer. Otherwise result is put into tmp_buf, which should be
5778 twice of the vector size.
5779
5780 \param vect1 - operand 1
5781 \param vect2 - operand 2
5782 \param tmp_buf - pointer on temporary buffer
5783 \param dsize - out destination size
5784
5785 \return Result pointer (tmp_buf)
5786
5787 @ingroup gapfunc
5788 */
5789 inline
5790 gap_word_t* gap_operation_or(const gap_word_t* BMRESTRICT vect1,
5791 const gap_word_t* BMRESTRICT vect2,
5792 gap_word_t* BMRESTRICT tmp_buf,
5793 unsigned& dsize) BMNOEXCEPT
5794 {
5795 gap_buff_op(tmp_buf, vect1, 1, vect2, 1, bm::and_op, dsize);
5796 gap_invert(tmp_buf);
5797 return tmp_buf;
5798 }
5799
5800 /*!
5801 \brief GAP bitcount OR operation test.
5802
5803 \param vect1 - operand 1
5804 \param vect2 - operand 2
5805 \return bitcount of vect1 OR vect2
5806
5807 @ingroup gapfunc
5808 */
5809 BMFORCEINLINE
5810 unsigned gap_count_or(const gap_word_t* BMRESTRICT vect1,
5811 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5812 {
5813 return gap_buff_count_op(vect1, vect2, bm::or_op);
5814 }
5815
5816
5817
5818 /*!
5819 \brief GAP SUB (AND NOT) operation.
5820
5821 Function performs SUB logical operation on gap vectors.
5822 If possible function put the result into vect1 and returns this
5823 pointer. Otherwise result is put into tmp_buf, which should be
5824 twice of the vector size.
5825
5826 \param vect1 - operand 1
5827 \param vect2 - operand 2
5828 \param tmp_buf - pointer on temporary buffer
5829 \param dsize - out destination size
5830
5831 \return Result pointer (tmp_buf)
5832
5833 @ingroup gapfunc
5834 */
5835 inline gap_word_t* gap_operation_sub(const gap_word_t* BMRESTRICT vect1,
5836 const gap_word_t* BMRESTRICT vect2,
5837 gap_word_t* BMRESTRICT tmp_buf,
5838 unsigned& dsize) BMNOEXCEPT
5839 {
5840 gap_buff_op(tmp_buf, vect1, 0, vect2, 1, and_op, dsize);
5841 return tmp_buf;
5842 }
5843
5844
5845 /*!
5846 \brief GAP SUB operation test.
5847
5848 Function performs AND logical operation on gap vectors.
5849 If possible function put the result into vect1 and returns this
5850 pointer. Otherwise result is put into tmp_buf, which should be
5851 twice of the vector size.
5852
5853 \param vect1 - operand 1
5854 \param vect2 - operand 2
5855 \return non zero value if operation returns any 1 bit
5856
5857 @ingroup gapfunc
5858 */
5859 BMFORCEINLINE
5860 unsigned gap_operation_any_sub(const gap_word_t* BMRESTRICT vect1,
5861 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5862 {
5863 return gap_buff_any_op(vect1, 0, vect2, 1, bm::and_op);
5864 }
5865
5866
5867 /*!
5868 \brief GAP bitcount SUB (AND NOT) operation test.
5869
5870 \param vect1 - operand 1
5871 \param vect2 - operand 2
5872 \return bitcount of vect1 SUB (AND NOT) vect2
5873
5874 @ingroup gapfunc
5875 */
5876 BMFORCEINLINE
5877 unsigned gap_count_sub(const gap_word_t* BMRESTRICT vect1,
5878 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5879 {
5880 return gap_buff_count_op(vect1, vect2, bm::sub_op);
5881 }
5882
5883
5884 // ----------------------------------------------------------------------
5885
5886 // BIT blocks manipulation functions:
5887
5888
5889 /*!
5890 \brief Bitblock copy operation.
5891
5892 \param dst - destination block.
5893 \param src - source block.
5894
5895 @ingroup bitfunc
5896 */
5897 inline
5898 void bit_block_copy(bm::word_t* BMRESTRICT dst,
5899 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5900 {
5901 #ifdef BMVECTOPT
5902 VECT_COPY_BLOCK(dst, src);
5903 #else
5904 ::memcpy(dst, src, bm::set_block_size * sizeof(bm::word_t));
5905 #endif
5906 }
5907
5908 /*!
5909 \brief Bitblock copy/stream operation.
5910
5911 \param dst - destination block.
5912 \param src - source block.
5913
5914 @ingroup bitfunc
5915 */
5916 inline
5917 void bit_block_stream(bm::word_t* BMRESTRICT dst,
5918 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5919 {
5920 #ifdef VECT_STREAM_BLOCK
5921 VECT_STREAM_BLOCK(dst, src);
5922 #else
5923 ::memcpy(dst, src, bm::set_block_size * sizeof(bm::word_t));
5924 #endif
5925 }
5926
5927
5928 /*!
5929 \brief Plain bitblock AND operation.
5930 Function does not analyse availability of source and destination blocks.
5931
5932 \param dst - destination block.
5933 \param src - source block.
5934
5935 \return 0 if AND operation did not produce anything (no 1s in the output)
5936
5937 @ingroup bitfunc
5938 */
5939 inline
5940 bm::id64_t bit_block_and(bm::word_t* BMRESTRICT dst,
5941 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5942 {
5943 BM_ASSERT(dst);
5944 BM_ASSERT(src);
5945 BM_ASSERT(dst != src);
5946
5947 #ifdef BMVECTOPT
5948 bm::id64_t acc = VECT_AND_BLOCK(dst, src);
5949 #else
5950 unsigned arr_sz = bm::set_block_size / 2;
5951
5952 const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)src;
5953 bm::bit_block_t::bunion_t* BMRESTRICT dst_u = (bm::bit_block_t::bunion_t*)dst;
5954
5955 bm::id64_t acc = 0;
5956 for (unsigned i = 0; i < arr_sz; i+=4)
5957 {
5958 acc |= dst_u->w64[i] &= src_u->w64[i];
5959 acc |= dst_u->w64[i+1] &= src_u->w64[i+1];
5960 acc |= dst_u->w64[i+2] &= src_u->w64[i+2];
5961 acc |= dst_u->w64[i+3] &= src_u->w64[i+3];
5962 }
5963 #endif
5964 return acc;
5965 }
5966
5967 /*!
5968 \brief digest based bit-block AND
5969
5970 \param dst - destination block.
5971 \param src - source block.
5972 \param digest - known digest of dst block
5973
5974 \return new digest
5975
5976 @ingroup bitfunc
5977 */
5978 inline
5979 bm::id64_t bit_block_and(bm::word_t* BMRESTRICT dst,
5980 const bm::word_t* BMRESTRICT src,
5981 bm::id64_t digest) BMNOEXCEPT
5982 {
5983 BM_ASSERT(dst);
5984 BM_ASSERT(src);
5985 BM_ASSERT(dst != src);
5986
5987 const bm::id64_t mask(1ull);
5988 bm::id64_t d = digest;
5989 while (d)
5990 {
5991 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
5992
5993 unsigned wave = bm::word_bitcount64(t - 1);
5994 unsigned off = wave * bm::set_block_digest_wave_size;
5995
5996 #if defined(VECT_AND_DIGEST)
5997 bool all_zero = VECT_AND_DIGEST(&dst[off], &src[off]);
5998 if (all_zero)
5999 digest &= ~(mask << wave);
6000 #else
6001 const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)(&src[off]);
6002 bm::bit_block_t::bunion_t* BMRESTRICT dst_u = (bm::bit_block_t::bunion_t*)(&dst[off]);
6003
6004 bm::id64_t acc = 0;
6005 unsigned j = 0;
6006 do
6007 {
6008 acc |= dst_u->w64[j+0] &= src_u->w64[j+0];
6009 acc |= dst_u->w64[j+1] &= src_u->w64[j+1];
6010 acc |= dst_u->w64[j+2] &= src_u->w64[j+2];
6011 acc |= dst_u->w64[j+3] &= src_u->w64[j+3];
6012 j+=4;
6013 } while (j < bm::set_block_digest_wave_size/2);
6014
6015 if (!acc) // all zero
6016 digest &= ~(mask << wave);
6017 #endif
6018
6019 d = bm::bmi_bslr_u64(d); // d &= d - 1;
6020 } // while
6021
6022 return digest;
6023 }
6024
6025
6026 /*!
6027 \brief digest based bit-block AND 5-way
6028
6029 \return new digest
6030
6031 @ingroup bitfunc
6032 */
6033 inline
6034 bm::id64_t bit_block_and_5way(bm::word_t* BMRESTRICT dst,
6035 const bm::word_t* BMRESTRICT src0,
6036 const bm::word_t* BMRESTRICT src1,
6037 const bm::word_t* BMRESTRICT src2,
6038 const bm::word_t* BMRESTRICT src3,
6039 bm::id64_t digest) BMNOEXCEPT
6040 {
6041 BM_ASSERT(dst);
6042 BM_ASSERT(src0 && src1 && src2 && src3);
6043
6044 const bm::id64_t mask(1ull);
6045 bm::id64_t d = digest;
6046 while (d)
6047 {
6048 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
6049
6050 unsigned wave = bm::word_bitcount64(t - 1);
6051 unsigned off = wave * bm::set_block_digest_wave_size;
6052
6053 #if defined(VECT_AND_DIGEST_5WAY)
6054 bool all_zero = VECT_AND_DIGEST_5WAY(&dst[off], &src0[off], &src1[off], &src2[off], &src3[off]);
6055 if (all_zero)
6056 digest &= ~(mask << wave);
6057 #else
6058 const bm::bit_block_t::bunion_t* BMRESTRICT src_u0 = (const bm::bit_block_t::bunion_t*)(&src0[off]);
6059 const bm::bit_block_t::bunion_t* BMRESTRICT src_u1 = (const bm::bit_block_t::bunion_t*)(&src1[off]);
6060 const bm::bit_block_t::bunion_t* BMRESTRICT src_u2 = (const bm::bit_block_t::bunion_t*)(&src2[off]);
6061 const bm::bit_block_t::bunion_t* BMRESTRICT src_u3 = (const bm::bit_block_t::bunion_t*)(&src3[off]);
6062 bm::bit_block_t::bunion_t* BMRESTRICT dst_u = (bm::bit_block_t::bunion_t*)(&dst[off]);
6063
6064 bm::id64_t acc = 0;
6065 unsigned j = 0;
6066 do
6067 {
6068 acc |= dst_u->w64[j + 0] &= src_u0->w64[j + 0] & src_u1->w64[j + 0] & src_u2->w64[j + 0] & src_u3->w64[j + 0];
6069 acc |= dst_u->w64[j + 1] &= src_u0->w64[j + 1] & src_u1->w64[j + 1] & src_u2->w64[j + 1] & src_u3->w64[j + 1];
6070 acc |= dst_u->w64[j + 2] &= src_u0->w64[j + 2] & src_u1->w64[j + 2] & src_u2->w64[j + 2] & src_u3->w64[j + 2];
6071 acc |= dst_u->w64[j + 3] &= src_u0->w64[j + 3] & src_u1->w64[j + 3] & src_u2->w64[j + 3] & src_u3->w64[j + 3];
6072 j += 4;
6073 } while (j < bm::set_block_digest_wave_size / 2);
6074
6075 if (!acc) // all zero
6076 digest &= ~(mask << wave);
6077 #endif
6078
6079 d = bm::bmi_bslr_u64(d); // d &= d - 1;
6080 } // while
6081
6082 return digest;
6083 }
6084
6085
6086 /*!
6087 \brief digest based bit-block AND
6088
6089 dst = src1 AND src2
6090
6091 \param dst - destination block.
6092 \param src1 - source block.
6093 \param src2 - source block.
6094 \param digest - known initial digest
6095
6096 \return new digest
6097
6098 @ingroup bitfunc
6099 */
6100 inline
6101 bm::id64_t bit_block_and_2way(bm::word_t* BMRESTRICT dst,
6102 const bm::word_t* BMRESTRICT src1,
6103 const bm::word_t* BMRESTRICT src2,
6104 bm::id64_t digest) BMNOEXCEPT
6105 {
6106 BM_ASSERT(dst);
6107 BM_ASSERT(src1 && src2);
6108 BM_ASSERT(dst != src1 && dst != src2);
6109
6110 const bm::id64_t mask(1ull);
6111 bm::id64_t d = digest;
6112 while (d)
6113 {
6114 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
6115
6116 unsigned wave = bm::word_bitcount64(t - 1);
6117 unsigned off = wave * bm::set_block_digest_wave_size;
6118
6119 #if defined(VECT_AND_DIGEST_2WAY)
6120 bool all_zero = VECT_AND_DIGEST_2WAY(&dst[off], &src1[off], &src2[off]);
6121 if (all_zero)
6122 digest &= ~(mask << wave);
6123 #else
6124 const bm::bit_block_t::bunion_t* BMRESTRICT src_u1 =
6125 (const bm::bit_block_t::bunion_t*)(&src1[off]);
6126 const bm::bit_block_t::bunion_t* BMRESTRICT src_u2 =
6127 (const bm::bit_block_t::bunion_t*)(&src2[off]);
6128 bm::bit_block_t::bunion_t* BMRESTRICT dst_u =
6129 (bm::bit_block_t::bunion_t*)(&dst[off]);
6130
6131 bm::id64_t acc = 0;
6132 unsigned j = 0;
6133 do
6134 {
6135 acc |= dst_u->w64[j+0] = src_u1->w64[j+0] & src_u2->w64[j+0];
6136 acc |= dst_u->w64[j+1] = src_u1->w64[j+1] & src_u2->w64[j+1];
6137 acc |= dst_u->w64[j+2] = src_u1->w64[j+2] & src_u2->w64[j+2];
6138 acc |= dst_u->w64[j+3] = src_u1->w64[j+3] & src_u2->w64[j+3];
6139 j+=4;
6140 } while (j < bm::set_block_digest_wave_size/2);
6141
6142 if (!acc) // all zero
6143 digest &= ~(mask << wave);
6144 #endif
6145
6146 d = bm::bmi_bslr_u64(d); // d &= d - 1;
6147 } // while
6148
6149 return digest;
6150 }
6151
6152
6153
6154 /*!
6155 \brief Function ANDs two bitblocks and computes the bitcount.
6156 Function does not analyse availability of source blocks.
6157
6158 \param src1 - first bit block
6159 \param src2 - second bit block
6160
6161 @ingroup bitfunc
6162 */
6163 inline
6164 unsigned bit_block_and_count(const bm::word_t* BMRESTRICT src1,
6165 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6166 {
6167 unsigned count;
6168 const bm::word_t* src1_end = src1 + bm::set_block_size;
6169 #ifdef BMVECTOPT
6170 count = VECT_BITCOUNT_AND(src1, src1_end, src2);
6171 #else
6172 count = 0;
6173 # ifdef BM64OPT
6174 const bm::id64_t* b1 = (bm::id64_t*) src1;
6175 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6176 const bm::id64_t* b2 = (bm::id64_t*) src2;
6177 do
6178 {
6179 count += bitcount64_4way(b1[0] & b2[0],
6180 b1[1] & b2[1],
6181 b1[2] & b2[2],
6182 b1[3] & b2[3]);
6183 b1 += 4;
6184 b2 += 4;
6185 } while (b1 < b1_end);
6186 # else
6187 do
6188 {
6189 BM_INCWORD_BITCOUNT(count, src1[0] & src2[0]);
6190 BM_INCWORD_BITCOUNT(count, src1[1] & src2[1]);
6191 BM_INCWORD_BITCOUNT(count, src1[2] & src2[2]);
6192 BM_INCWORD_BITCOUNT(count, src1[3] & src2[3]);
6193
6194 src1+=4;
6195 src2+=4;
6196 } while (src1 < src1_end);
6197 # endif
6198 #endif
6199 return count;
6200 }
6201
6202
6203 /*!
6204 \brief Function ANDs two bitblocks and tests for any bit.
6205 Function does not analyse availability of source blocks.
6206
6207 \param src1 - first bit block
6208 \param src2 - second bit block
6209
6210 @ingroup bitfunc
6211 */
6212 inline
6213 unsigned bit_block_and_any(const bm::word_t* src1,
6214 const bm::word_t* src2) BMNOEXCEPT
6215 {
6216 unsigned count = 0;
6217 const bm::word_t* src1_end = src1 + bm::set_block_size;
6218 do
6219 {
6220 count = (src1[0] & src2[0]) |
6221 (src1[1] & src2[1]) |
6222 (src1[2] & src2[2]) |
6223 (src1[3] & src2[3]);
6224
6225 src1+=4; src2+=4;
6226 } while ((src1 < src1_end) && !count);
6227 return count;
6228 }
6229
6230
6231
6232
6233 /*!
6234 \brief Function XORs two bitblocks and computes the bitcount.
6235 Function does not analyse availability of source blocks.
6236
6237 \param src1 - first bit block
6238 \param src2 - second bit block
6239
6240 @ingroup bitfunc
6241 */
6242 inline
6243 unsigned bit_block_xor_count(const bm::word_t* BMRESTRICT src1,
6244 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6245 {
6246 unsigned count;
6247 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6248 #ifdef BMVECTOPT
6249 count = VECT_BITCOUNT_XOR(src1, src1_end, src2);
6250 #else
6251 count = 0;
6252 # ifdef BM64OPT
6253 const bm::id64_t* b1 = (bm::id64_t*) src1;
6254 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6255 const bm::id64_t* b2 = (bm::id64_t*) src2;
6256 do
6257 {
6258 count += bitcount64_4way(b1[0] ^ b2[0],
6259 b1[1] ^ b2[1],
6260 b1[2] ^ b2[2],
6261 b1[3] ^ b2[3]);
6262 b1 += 4;
6263 b2 += 4;
6264 } while (b1 < b1_end);
6265 # else
6266 do
6267 {
6268 BM_INCWORD_BITCOUNT(count, src1[0] ^ src2[0]);
6269 BM_INCWORD_BITCOUNT(count, src1[1] ^ src2[1]);
6270 BM_INCWORD_BITCOUNT(count, src1[2] ^ src2[2]);
6271 BM_INCWORD_BITCOUNT(count, src1[3] ^ src2[3]);
6272
6273 src1+=4;
6274 src2+=4;
6275 } while (src1 < src1_end);
6276 # endif
6277 #endif
6278 return count;
6279 }
6280
6281
6282 /*!
6283 \brief Function XORs two bitblocks and and tests for any bit.
6284 Function does not analyse availability of source blocks.
6285
6286 \param src1 - first bit block.
6287 \param src2 - second bit block.
6288
6289 @ingroup bitfunc
6290 */
6291 inline
6292 unsigned bit_block_xor_any(const bm::word_t* BMRESTRICT src1,
6293 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6294 {
6295 unsigned count = 0;
6296 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6297 do
6298 {
6299 count = (src1[0] ^ src2[0]) |
6300 (src1[1] ^ src2[1]) |
6301 (src1[2] ^ src2[2]) |
6302 (src1[3] ^ src2[3]);
6303
6304 src1+=4; src2+=4;
6305 } while (!count && (src1 < src1_end));
6306 return count;
6307 }
6308
6309 /*!
6310 \brief Function SUBs two bitblocks and computes the bitcount.
6311 Function does not analyse availability of source blocks.
6312
6313 \param src1 - first bit block.
6314 \param src2 - second bit block.
6315
6316 @ingroup bitfunc
6317 */
6318 inline
6319 unsigned bit_block_sub_count(const bm::word_t* BMRESTRICT src1,
6320 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6321 {
6322 unsigned count;
6323 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6324 #ifdef BMVECTOPT
6325 count = VECT_BITCOUNT_SUB(src1, src1_end, src2);
6326 #else
6327 count = 0;
6328 # ifdef BM64OPT
6329 const bm::id64_t* b1 = (bm::id64_t*) src1;
6330 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6331 const bm::id64_t* b2 = (bm::id64_t*) src2;
6332 do
6333 {
6334 count += bitcount64_4way(b1[0] & ~b2[0],
6335 b1[1] & ~b2[1],
6336 b1[2] & ~b2[2],
6337 b1[3] & ~b2[3]);
6338 b1 += 4;
6339 b2 += 4;
6340 } while (b1 < b1_end);
6341 # else
6342 do
6343 {
6344 BM_INCWORD_BITCOUNT(count, src1[0] & ~src2[0]);
6345 BM_INCWORD_BITCOUNT(count, src1[1] & ~src2[1]);
6346 BM_INCWORD_BITCOUNT(count, src1[2] & ~src2[2]);
6347 BM_INCWORD_BITCOUNT(count, src1[3] & ~src2[3]);
6348
6349 src1+=4;
6350 src2+=4;
6351 } while (src1 < src1_end);
6352 # endif
6353 #endif
6354 return count;
6355 }
6356
6357 /*!
6358 \brief Function SUBs two bitblocks and and tests for any bit.
6359 Function does not analyse availability of source blocks.
6360
6361 \param src1 - first bit block.
6362 \param src2 - second bit block.
6363
6364 @ingroup bitfunc
6365 */
6366 inline
6367 unsigned bit_block_sub_any(const bm::word_t* BMRESTRICT src1,
6368 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6369 {
6370 unsigned count = 0;
6371 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6372
6373 do
6374 {
6375 count = (src1[0] & ~src2[0]) |
6376 (src1[1] & ~src2[1]) |
6377 (src1[2] & ~src2[2]) |
6378 (src1[3] & ~src2[3]);
6379
6380 src1+=4; src2+=4;
6381 } while ((src1 < src1_end) && (count == 0));
6382 return count;
6383 }
6384
6385
6386 /*!
6387 \brief Function ORs two bitblocks and computes the bitcount.
6388 Function does not analyse availability of source blocks.
6389
6390 \param src1 - first bit block
6391 \param src2 - second bit block.
6392
6393 @ingroup bitfunc
6394 */
6395 inline
6396 unsigned bit_block_or_count(const bm::word_t* src1,
6397 const bm::word_t* src2) BMNOEXCEPT
6398 {
6399 unsigned count;
6400 const bm::word_t* src1_end = src1 + bm::set_block_size;
6401 #ifdef BMVECTOPT
6402 count = VECT_BITCOUNT_OR(src1, src1_end, src2);
6403 #else
6404 count = 0;
6405 # ifdef BM64OPT
6406 const bm::id64_t* b1 = (bm::id64_t*) src1;
6407 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6408 const bm::id64_t* b2 = (bm::id64_t*) src2;
6409 do
6410 {
6411 count += bitcount64_4way(b1[0] | b2[0],
6412 b1[1] | b2[1],
6413 b1[2] | b2[2],
6414 b1[3] | b2[3]);
6415 b1 += 4;
6416 b2 += 4;
6417 } while (b1 < b1_end);
6418 # else
6419 do
6420 {
6421 BM_INCWORD_BITCOUNT(count, src1[0] | src2[0]);
6422 BM_INCWORD_BITCOUNT(count, src1[1] | src2[1]);
6423 BM_INCWORD_BITCOUNT(count, src1[2] | src2[2]);
6424 BM_INCWORD_BITCOUNT(count, src1[3] | src2[3]);
6425
6426 src1+=4;
6427 src2+=4;
6428 } while (src1 < src1_end);
6429 # endif
6430 #endif
6431 return count;
6432 }
6433
6434 /*!
6435 \brief Function ORs two bitblocks and and tests for any bit.
6436 Function does not analyse availability of source blocks.
6437
6438 \param src1 - first bit block.
6439 \param src2 - second bit block.
6440
6441 @ingroup bitfunc
6442 */
6443 inline
6444 unsigned bit_block_or_any(const bm::word_t* BMRESTRICT src1,
6445 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6446 {
6447 unsigned count = 0;
6448 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6449 do
6450 {
6451 count = (src1[0] | src2[0]) |
6452 (src1[1] | src2[1]) |
6453 (src1[2] | src2[2]) |
6454 (src1[3] | src2[3]);
6455
6456 src1+=4; src2+=4;
6457 } while (!count && (src1 < src1_end));
6458 return count;
6459 }
6460
6461
6462
6463
6464 /*!
6465 \brief bitblock AND operation.
6466
6467 \param dst - destination block.
6468 \param src - source block.
6469
6470 \returns pointer on destination block.
6471 If returned value equal to src means that block mutation requested.
6472 NULL is valid return value.
6473
6474 @ingroup bitfunc
6475 */
6476 inline bm::word_t* bit_operation_and(bm::word_t* BMRESTRICT dst,
6477 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6478 {
6479 BM_ASSERT(dst || src);
6480
6481 bm::word_t* ret = dst;
6482
6483 if (IS_VALID_ADDR(dst)) // The destination block already exists
6484 {
6485 if (!IS_VALID_ADDR(src))
6486 {
6487 if (IS_EMPTY_BLOCK(src))
6488 {
6489 //If the source block is zero
6490 //just clean the destination block
6491 return 0;
6492 }
6493 }
6494 else
6495 {
6496 // Regular operation AND on the whole block.
6497 //
6498 auto any = bm::bit_block_and(dst, src);
6499 if (!any)
6500 ret = 0;
6501 }
6502 }
6503 else // The destination block does not exist yet
6504 {
6505 if(!IS_VALID_ADDR(src))
6506 {
6507 if(IS_EMPTY_BLOCK(src))
6508 {
6509 // The source block is empty.
6510 // One argument empty - all result is empty.
6511 return 0;
6512 }
6513 // Here we have nothing to do.
6514 // Src block is all ON, dst block remains as it is
6515 }
6516 else // destination block does not exists, src - valid block
6517 {
6518 if (IS_FULL_BLOCK(dst))
6519 return const_cast<bm::word_t*>(src);
6520 // Nothng to do.
6521 // Dst block is all ZERO no combination required.
6522 }
6523 }
6524
6525 return ret;
6526 }
6527
6528
6529 /*!
6530 \brief Performs bitblock AND operation and calculates bitcount of the result.
6531
6532 \param src1 - first bit block.
6533 \param src2 - second bit block.
6534
6535 \returns bitcount value
6536
6537 @ingroup bitfunc
6538 */
6539 inline
6540 bm::id_t bit_operation_and_count(const bm::word_t* BMRESTRICT src1,
6541 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6542 {
6543 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
6544 return 0;
6545 if (src1 == FULL_BLOCK_FAKE_ADDR)
6546 src1 = FULL_BLOCK_REAL_ADDR;
6547 if (src2 == FULL_BLOCK_FAKE_ADDR)
6548 src2 = FULL_BLOCK_REAL_ADDR;
6549
6550 return bit_block_and_count(src1, src2);
6551 }
6552
6553 /*!
6554 \brief Performs bitblock AND operation test.
6555
6556 \param src1 - first bit block.
6557 \param src2 - second bit block.
6558
6559 \returns non zero if there is any value
6560
6561 @ingroup bitfunc
6562 */
6563 inline
6564 bm::id_t bit_operation_and_any(const bm::word_t* BMRESTRICT src1,
6565 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6566 {
6567 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
6568 return 0;
6569 if (src1 == FULL_BLOCK_FAKE_ADDR)
6570 src1 = FULL_BLOCK_REAL_ADDR;
6571 if (src2 == FULL_BLOCK_FAKE_ADDR)
6572 src2 = FULL_BLOCK_REAL_ADDR;
6573 return bit_block_and_any(src1, src2);
6574 }
6575
6576
6577
6578 /*!
6579 \brief Performs bitblock SUB operation and calculates bitcount of the result.
6580
6581 \param src1 - first bit block.
6582 \param src2 - second bit block
6583
6584 \returns bitcount value
6585
6586 @ingroup bitfunc
6587 */
6588 inline
6589 bm::id_t bit_operation_sub_count(const bm::word_t* BMRESTRICT src1,
6590 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6591 {
6592 if (src1 == src2)
6593 return 0;
6594 if (IS_EMPTY_BLOCK(src1))
6595 return 0;
6596 if (IS_EMPTY_BLOCK(src2)) // nothing to diff
6597 {
6598 if (IS_FULL_BLOCK(src1))
6599 return bm::gap_max_bits;
6600 return bm::bit_block_count(src1);
6601 }
6602 if (IS_FULL_BLOCK(src2))
6603 return 0;
6604
6605 if (src1 == FULL_BLOCK_FAKE_ADDR)
6606 src1 = FULL_BLOCK_REAL_ADDR;
6607 if (src2 == FULL_BLOCK_FAKE_ADDR)
6608 src2 = FULL_BLOCK_REAL_ADDR;
6609
6610 return bm::bit_block_sub_count(src1, src2);
6611 }
6612
6613
6614 /*!
6615 \brief Performs inverted bitblock SUB operation and calculates
6616 bitcount of the result.
6617
6618 \param src1 - first bit block.
6619 \param src2 - second bit block
6620
6621 \returns bitcount value
6622
6623 @ingroup bitfunc
6624 */
6625 inline
6626 bm::id_t bit_operation_sub_count_inv(const bm::word_t* BMRESTRICT src1,
6627 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6628 {
6629 return bit_operation_sub_count(src2, src1);
6630 }
6631
6632
6633 /*!
6634 \brief Performs bitblock test of SUB operation.
6635
6636 \param src1 - first bit block.
6637 \param src2 - second bit block
6638
6639 \returns non zero value if there are any bits
6640
6641 @ingroup bitfunc
6642 */
6643 inline
6644 bm::id_t bit_operation_sub_any(const bm::word_t* BMRESTRICT src1,
6645 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6646 {
6647 if (IS_EMPTY_BLOCK(src1))
6648 return 0;
6649 if (src1 == src2)
6650 return 0;
6651
6652 if (IS_EMPTY_BLOCK(src2)) // nothing to diff
6653 return !bit_is_all_zero(src1);
6654
6655 if (IS_FULL_BLOCK(src2))
6656 return 0;
6657
6658 if (src1 == FULL_BLOCK_FAKE_ADDR)
6659 src1 = FULL_BLOCK_REAL_ADDR;
6660 if (src2 == FULL_BLOCK_FAKE_ADDR)
6661 src2 = FULL_BLOCK_REAL_ADDR;
6662
6663 return bm::bit_block_sub_any(src1, src2);
6664 }
6665
6666
6667
6668 /*!
6669 \brief Performs bitblock OR operation and calculates bitcount of the result.
6670
6671 \param src1 - first bit block.
6672 \param src2 - second bit block.
6673
6674 \returns bitcount value
6675
6676 @ingroup bitfunc
6677 */
6678 inline
6679 bm::id_t bit_operation_or_count(const bm::word_t* BMRESTRICT src1,
6680 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6681 {
6682 if (IS_FULL_BLOCK(src1) || IS_FULL_BLOCK(src2))
6683 return bm::gap_max_bits;
6684
6685 if (IS_EMPTY_BLOCK(src1))
6686 {
6687 if (!IS_EMPTY_BLOCK(src2))
6688 return bm::bit_block_count(src2);
6689 else
6690 return 0; // both blocks are empty
6691 }
6692 else
6693 {
6694 if (IS_EMPTY_BLOCK(src2))
6695 return bm::bit_block_count(src1);
6696 }
6697 if (src1 == FULL_BLOCK_FAKE_ADDR)
6698 src1 = FULL_BLOCK_REAL_ADDR;
6699 if (src2 == FULL_BLOCK_FAKE_ADDR)
6700 src2 = FULL_BLOCK_REAL_ADDR;
6701
6702 return bm::bit_block_or_count(src1, src2);
6703 }
6704
6705 /*!
6706 \brief Performs bitblock OR operation test.
6707
6708 \param src1 - first bit block.
6709 \param src2 - second bit block.
6710
6711 \returns non zero value if there are any bits
6712
6713 @ingroup bitfunc
6714 */
6715 inline
6716 bm::id_t bit_operation_or_any(const bm::word_t* BMRESTRICT src1,
6717 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6718 {
6719 if (IS_EMPTY_BLOCK(src1))
6720 {
6721 if (!IS_EMPTY_BLOCK(src2))
6722 return !bit_is_all_zero(src2);
6723 else
6724 return 0; // both blocks are empty
6725 }
6726 else
6727 {
6728 if (IS_EMPTY_BLOCK(src2))
6729 return !bit_is_all_zero(src1);
6730 }
6731 if (src1 == FULL_BLOCK_FAKE_ADDR)
6732 src1 = FULL_BLOCK_REAL_ADDR;
6733 if (src2 == FULL_BLOCK_FAKE_ADDR)
6734 src2 = FULL_BLOCK_REAL_ADDR;
6735
6736 return bit_block_or_any(src1, src2);
6737 }
6738
6739 /*!
6740 \brief Plain bitblock OR operation.
6741 Function does not analyse availability of source and destination blocks.
6742
6743 \param dst - destination block.
6744 \param src - source block.
6745
6746 @ingroup bitfunc
6747 */
6748 inline
6749 bool bit_block_or(bm::word_t* BMRESTRICT dst,
6750 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6751 {
6752 #ifdef BMVECTOPT
6753 return VECT_OR_BLOCK(dst, src);
6754 #else
6755 const bm::wordop_t* BMRESTRICT wrd_ptr = (wordop_t*)src;
6756 const bm::wordop_t* BMRESTRICT wrd_end = (wordop_t*)(src + bm::set_block_size);
6757 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6758
6759 bm::wordop_t acc = 0;
6760 const bm::wordop_t not_acc = acc = ~acc;
6761
6762 do
6763 {
6764 acc &= (dst_ptr[0] |= wrd_ptr[0]);
6765 acc &= (dst_ptr[1] |= wrd_ptr[1]);
6766 acc &= (dst_ptr[2] |= wrd_ptr[2]);
6767 acc &= (dst_ptr[3] |= wrd_ptr[3]);
6768
6769 dst_ptr+=4;wrd_ptr+=4;
6770
6771 } while (wrd_ptr < wrd_end);
6772 return acc == not_acc;
6773 #endif
6774 }
6775
6776 /*!
6777 \brief 2 way (target := source1 | source2) bitblock OR operation.
6778 \param dst - dest block [out]
6779 \param src1 - source 1
6780 \param src2 - source 2
6781
6782 @return 1 if produced block of ALL ones
6783 @ingroup bitfunc
6784 */
6785 inline
6786 bool bit_block_or_2way(bm::word_t* BMRESTRICT dst,
6787 const bm::word_t* BMRESTRICT src1,
6788 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6789 {
6790 #ifdef BMVECTOPT
6791 return VECT_OR_BLOCK_2WAY(dst, src1, src2);
6792 #else
6793 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6794 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6795 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6796 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6797
6798 bm::wordop_t acc = 0;
6799 const bm::wordop_t not_acc = acc = ~acc;
6800 do
6801 {
6802 acc &= (dst_ptr[0] = wrd_ptr1[0] | wrd_ptr2[0]);
6803 acc &= (dst_ptr[1] = wrd_ptr1[1] | wrd_ptr2[1]);
6804 acc &= (dst_ptr[2] = wrd_ptr1[2] | wrd_ptr2[2]);
6805 acc &= (dst_ptr[3] = wrd_ptr1[3] | wrd_ptr2[3]);
6806
6807 dst_ptr+=4; wrd_ptr1+=4; wrd_ptr2+=4;
6808
6809 } while (wrd_ptr1 < wrd_end1);
6810 return acc == not_acc;
6811 #endif
6812 }
6813
6814
6815 /*!
6816 \brief 2 way (target := source1 ^ source2) bitblock XOR operation.
6817 \param dst - dest block [out]
6818 \param src1 - source 1
6819 \param src2 - source 2
6820
6821 @return OR accumulator
6822 @ingroup bitfunc
6823 */
6824 inline
6825 bm::id64_t bit_block_xor_2way(bm::word_t* BMRESTRICT dst,
6826 const bm::word_t* BMRESTRICT src1,
6827 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6828 {
6829 #ifdef BMVECTOPT
6830 return VECT_XOR_BLOCK_2WAY(dst, src1, src2);
6831 #else
6832 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6833 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6834 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6835 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6836
6837 bm::wordop_t acc = 0;
6838 do
6839 {
6840 acc |= (dst_ptr[0] = wrd_ptr1[0] ^ wrd_ptr2[0]);
6841 acc |= (dst_ptr[1] = wrd_ptr1[1] ^ wrd_ptr2[1]);
6842 acc |= (dst_ptr[2] = wrd_ptr1[2] ^ wrd_ptr2[2]);
6843 acc |= (dst_ptr[3] = wrd_ptr1[3] ^ wrd_ptr2[3]);
6844
6845 dst_ptr+=4; wrd_ptr1+=4; wrd_ptr2+=4;
6846
6847 } while (wrd_ptr1 < wrd_end1);
6848 return acc;
6849 #endif
6850 }
6851
6852
6853 /*!
6854 \brief 3 way (target | source1 | source2) bitblock OR operation.
6855 Function does not analyse availability of source and destination blocks.
6856
6857 \param dst - sst-dest block [in,out]
6858 \param src1 - source 1
6859 \param src2 - source 2
6860
6861 @return 1 if produced block of ALL ones
6862
6863 @ingroup bitfunc
6864 */
6865 inline
6866 bool bit_block_or_3way(bm::word_t* BMRESTRICT dst,
6867 const bm::word_t* BMRESTRICT src1,
6868 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6869 {
6870 #ifdef BMVECTOPT
6871 return VECT_OR_BLOCK_3WAY(dst, src1, src2);
6872 #else
6873 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6874 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6875 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6876 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6877
6878 bm::wordop_t acc = 0;
6879 const bm::wordop_t not_acc = acc = ~acc;
6880 do
6881 {
6882 acc &= (dst_ptr[0] |= wrd_ptr1[0] | wrd_ptr2[0]);
6883 acc &= (dst_ptr[1] |= wrd_ptr1[1] | wrd_ptr2[1]);
6884 acc &= (dst_ptr[2] |= wrd_ptr1[2] | wrd_ptr2[2]);
6885 acc &= (dst_ptr[3] |= wrd_ptr1[3] | wrd_ptr2[3]);
6886
6887 dst_ptr+=4; wrd_ptr1+=4;wrd_ptr2+=4;
6888
6889 } while (wrd_ptr1 < wrd_end1);
6890 return acc == not_acc;
6891 #endif
6892 }
6893
6894
6895 /*!
6896 \brief 5 way (target, source1, source2) bitblock OR operation.
6897 Function does not analyse availability of source and destination blocks.
6898
6899 \param dst - destination block.
6900 \param src1 - source1, etc
6901 \param src2 - source1, etc
6902 \param src3 - source1, etc
6903 \param src4 - source1, etc
6904
6905 @return 1 if produced block of ALL ones
6906
6907 @ingroup bitfunc
6908 */
6909 inline
6910 bool bit_block_or_5way(bm::word_t* BMRESTRICT dst,
6911 const bm::word_t* BMRESTRICT src1,
6912 const bm::word_t* BMRESTRICT src2,
6913 const bm::word_t* BMRESTRICT src3,
6914 const bm::word_t* BMRESTRICT src4) BMNOEXCEPT
6915 {
6916 #ifdef BMVECTOPT
6917 return VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4);
6918 #else
6919 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6920 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6921 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6922 const bm::wordop_t* BMRESTRICT wrd_ptr3 = (wordop_t*)src3;
6923 const bm::wordop_t* BMRESTRICT wrd_ptr4 = (wordop_t*)src4;
6924 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6925
6926 bm::wordop_t acc = 0;
6927 const bm::wordop_t not_acc = acc = ~acc;
6928 do
6929 {
6930 acc &= (dst_ptr[0] |= wrd_ptr1[0] | wrd_ptr2[0] | wrd_ptr3[0] | wrd_ptr4[0]);
6931 acc &= (dst_ptr[1] |= wrd_ptr1[1] | wrd_ptr2[1] | wrd_ptr3[1] | wrd_ptr4[1]);
6932 acc &= (dst_ptr[2] |= wrd_ptr1[2] | wrd_ptr2[2] | wrd_ptr3[2] | wrd_ptr4[2]);
6933 acc &= (dst_ptr[3] |= wrd_ptr1[3] | wrd_ptr2[3] | wrd_ptr3[3] | wrd_ptr4[3]);
6934
6935 dst_ptr+=4;
6936 wrd_ptr1+=4;wrd_ptr2+=4;wrd_ptr3+=4;wrd_ptr4+=4;
6937
6938 } while (wrd_ptr1 < wrd_end1);
6939 return acc == not_acc;
6940 #endif
6941 }
6942
6943
6944
6945
6946 /*!
6947 \brief Block OR operation. Makes analysis if block is 0 or FULL.
6948
6949 \param dst - destination block.
6950 \param src - source block.
6951
6952 \returns pointer on destination block.
6953 If returned value equal to src means that block mutation requested.
6954 NULL is valid return value.
6955
6956 @ingroup bitfunc
6957 */
6958 inline
6959 bm::word_t* bit_operation_or(bm::word_t* BMRESTRICT dst,
6960 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6961 {
6962 BM_ASSERT(dst || src);
6963
6964 bm::word_t* ret = dst;
6965
6966 if (IS_VALID_ADDR(dst)) // The destination block already exists
6967 {
6968 if (!IS_VALID_ADDR(src))
6969 {
6970 if (IS_FULL_BLOCK(src))
6971 {
6972 // if the source block is all set
6973 // just set the destination block
6974 ::memset(dst, 0xFF, bm::set_block_size * sizeof(bm::word_t));
6975 }
6976 }
6977 else
6978 {
6979 // Regular operation OR on the whole block
6980 bm::bit_block_or(dst, src);
6981 }
6982 }
6983 else // The destination block does not exist yet
6984 {
6985 if (!IS_VALID_ADDR(src))
6986 {
6987 if (IS_FULL_BLOCK(src))
6988 {
6989 // The source block is all set, because dst does not exist
6990 // we can simply replace it.
6991 return const_cast<bm::word_t*>(FULL_BLOCK_FAKE_ADDR);
6992 }
6993 }
6994 else
6995 {
6996 if (dst == 0)
6997 {
6998 // The only case when we have to allocate the new block:
6999 // Src is all zero and Dst does not exist
7000 return const_cast<bm::word_t*>(src);
7001 }
7002 }
7003 }
7004 return ret;
7005 }
7006
7007 /*!
7008 \brief Plain bitblock SUB (AND NOT) operation.
7009 Function does not analyse availability of source and destination blocks.
7010
7011 \param dst - destination block.
7012 \param src - source block.
7013
7014 \return 0 if SUB operation did not produce anything (no 1s in the output)
7015
7016 @ingroup bitfunc
7017 */
7018 inline
7019 bm::id64_t bit_block_sub(bm::word_t* BMRESTRICT dst,
7020 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7021 {
7022 #ifdef BMVECTOPT
7023 bm::id64_t acc = VECT_SUB_BLOCK(dst, src);
7024 return acc;
7025 #else
7026 unsigned arr_sz = bm::set_block_size / 2;
7027
7028 const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)src;
7029 bm::bit_block_t::bunion_t* BMRESTRICT dst_u = (bm::bit_block_t::bunion_t*)dst;
7030
7031 bm::id64_t acc = 0;
7032 for (unsigned i = 0; i < arr_sz; i+=4)
7033 {
7034 acc |= dst_u->w64[i] &= ~src_u->w64[i];
7035 acc |= dst_u->w64[i+1] &= ~src_u->w64[i+1];
7036 acc |= dst_u->w64[i+2] &= ~src_u->w64[i+2];
7037 acc |= dst_u->w64[i+3] &= ~src_u->w64[i+3];
7038 }
7039 return acc;
7040 #endif
7041 }
7042
7043 /*!
7044 \brief digest based bitblock SUB (AND NOT) operation
7045
7046 \param dst - destination block.
7047 \param src - source block.
7048 \param digest - known digest of dst block
7049
7050 \return new digest
7051
7052 @ingroup bitfunc
7053 */
7054 inline
7055 bm::id64_t bit_block_sub(bm::word_t* BMRESTRICT dst,
7056 const bm::word_t* BMRESTRICT src,
7057 bm::id64_t digest) BMNOEXCEPT
7058 {
7059 BM_ASSERT(dst);
7060 BM_ASSERT(src);
7061 BM_ASSERT(dst != src);
7062
7063 const bm::id64_t mask(1ull);
7064
7065 bm::id64_t d = digest;
7066 while (d)
7067 {
7068 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
7069
7070 unsigned wave = bm::word_bitcount64(t - 1);
7071 unsigned off = wave * bm::set_block_digest_wave_size;
7072
7073 #if defined(VECT_SUB_DIGEST)
7074 bool all_zero = VECT_SUB_DIGEST(&dst[off], &src[off]);
7075 if (all_zero)
7076 digest &= ~(mask << wave);
7077 #else
7078 const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)(&src[off]);
7079 bm::bit_block_t::bunion_t* BMRESTRICT dst_u = (bm::bit_block_t::bunion_t*)(&dst[off]);
7080
7081 bm::id64_t acc = 0;
7082 unsigned j = 0;
7083 do
7084 {
7085 acc |= dst_u->w64[j+0] &= ~src_u->w64[j+0];
7086 acc |= dst_u->w64[j+1] &= ~src_u->w64[j+1];
7087 acc |= dst_u->w64[j+2] &= ~src_u->w64[j+2];
7088 acc |= dst_u->w64[j+3] &= ~src_u->w64[j+3];
7089 j+=4;
7090 } while (j < bm::set_block_digest_wave_size/2);
7091
7092 if (!acc) // all zero
7093 digest &= ~(mask << wave);
7094 #endif
7095
7096 d = bm::bmi_bslr_u64(d); // d &= d - 1;
7097 } // while
7098
7099 return digest;
7100 }
7101
7102 /*!
7103 \brief digest based bitblock SUB (AND NOT) operation (3 operand)
7104
7105 \param dst - destination block.
7106 \param src1 - source block 1
7107 \param src2 - source block 2
7108 \param digest - known digest of dst block
7109
7110 \return new digest
7111
7112 @ingroup bitfunc
7113 */
7114 inline
7115 bm::id64_t bit_block_sub_2way(bm::word_t* BMRESTRICT dst,
7116 const bm::word_t* BMRESTRICT src1,
7117 const bm::word_t* BMRESTRICT src2,
7118 bm::id64_t digest) BMNOEXCEPT
7119 {
7120 BM_ASSERT(dst);
7121 BM_ASSERT(src1 && src2);
7122 BM_ASSERT(dst != src1 && dst != src2);
7123
7124 const bm::id64_t mask(1ull);
7125
7126 bm::id64_t d = digest;
7127 while (d)
7128 {
7129 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
7130
7131 unsigned wave = bm::word_bitcount64(t - 1);
7132 unsigned off = wave * bm::set_block_digest_wave_size;
7133
7134 #if defined(VECT_SUB_DIGEST_2WAY)
7135 bool all_zero = VECT_SUB_DIGEST_2WAY(&dst[off], &src1[off], &src2[off]);
7136 if (all_zero)
7137 digest &= ~(mask << wave);
7138 #else
7139 const bm::bit_block_t::bunion_t* BMRESTRICT src_u1 =
7140 (const bm::bit_block_t::bunion_t*)(&src1[off]);
7141 const bm::bit_block_t::bunion_t* BMRESTRICT src_u2 =
7142 (const bm::bit_block_t::bunion_t*)(&src2[off]);
7143 bm::bit_block_t::bunion_t* BMRESTRICT dst_u =
7144 (bm::bit_block_t::bunion_t*)(&dst[off]);
7145
7146 bm::id64_t acc = 0;
7147 unsigned j = 0;
7148 do
7149 {
7150 acc |= dst_u->w64[j+0] = src_u1->w64[j+0] & ~src_u2->w64[j+0];
7151 acc |= dst_u->w64[j+1] = src_u1->w64[j+1] & ~src_u2->w64[j+1];
7152 acc |= dst_u->w64[j+2] = src_u1->w64[j+2] & ~src_u2->w64[j+2];
7153 acc |= dst_u->w64[j+3] = src_u1->w64[j+3] & ~src_u2->w64[j+3];
7154 j+=4;
7155 } while (j < bm::set_block_digest_wave_size/2);
7156
7157 if (!acc) // all zero
7158 digest &= ~(mask << wave);
7159 #endif
7160
7161 d = bm::bmi_bslr_u64(d); // d &= d - 1;
7162 } // while
7163
7164 return digest;
7165 }
7166
7167
7168
7169 /*!
7170 \brief bitblock SUB operation.
7171
7172 \param dst - destination block.
7173 \param src - source block.
7174
7175 \returns pointer on destination block.
7176 If returned value equal to src means that block mutation requested.
7177 NULL is valid return value.
7178
7179 @ingroup bitfunc
7180 */
7181 inline
7182 bm::word_t* bit_operation_sub(bm::word_t* BMRESTRICT dst,
7183 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7184 {
7185 BM_ASSERT(dst || src);
7186
7187 bm::word_t* ret = dst;
7188 if (IS_VALID_ADDR(dst)) // The destination block already exists
7189 {
7190 if (!IS_VALID_ADDR(src))
7191 {
7192 if (IS_FULL_BLOCK(src))
7193 {
7194 // If the source block is all set
7195 // just clean the destination block
7196 return 0;
7197 }
7198 }
7199 else
7200 {
7201 auto any = bm::bit_block_sub(dst, src);
7202 if (!any)
7203 ret = 0;
7204 }
7205 }
7206 else // The destination block does not exist yet
7207 {
7208 if (!IS_VALID_ADDR(src))
7209 {
7210 if (IS_FULL_BLOCK(src))
7211 {
7212 // The source block is full
7213 return 0;
7214 }
7215 }
7216 else
7217 {
7218 if (IS_FULL_BLOCK(dst))
7219 {
7220 // The only case when we have to allocate the new block:
7221 // dst is all set and src exists
7222 return const_cast<bm::word_t*>(src);
7223 }
7224 }
7225 }
7226 return ret;
7227 }
7228
7229
7230 /*!
7231 \brief Plain bitblock XOR operation.
7232 Function does not analyse availability of source and destination blocks.
7233
7234 \param dst - destination block.
7235 \param src - source block.
7236
7237 @ingroup bitfunc
7238 */
7239 inline
7240 bm::id64_t bit_block_xor(bm::word_t* BMRESTRICT dst,
7241 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7242 {
7243 BM_ASSERT(dst);
7244 BM_ASSERT(src);
7245 BM_ASSERT(dst != src);
7246
7247 #ifdef BMVECTOPT
7248 bm::id64_t acc = VECT_XOR_BLOCK(dst, src);
7249 #else
7250 unsigned arr_sz = bm::set_block_size / 2;
7251
7252 const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)src;
7253 bm::bit_block_t::bunion_t* BMRESTRICT dst_u = (bm::bit_block_t::bunion_t*)dst;
7254
7255 bm::id64_t acc = 0;
7256 for (unsigned i = 0; i < arr_sz; i+=4)
7257 {
7258 acc |= dst_u->w64[i] ^= src_u->w64[i];
7259 acc |= dst_u->w64[i+1] ^= src_u->w64[i+1];
7260 acc |= dst_u->w64[i+2] ^= src_u->w64[i+2];
7261 acc |= dst_u->w64[i+3] ^= src_u->w64[i+3];
7262 }
7263 #endif
7264 return acc;
7265 }
7266
7267 /*!
7268 \brief bitblock AND NOT with constant ~0 mask operation.
7269
7270 \param dst - destination block.
7271 \param src - source block.
7272
7273 @ingroup bitfunc
7274 */
7275 inline
7276 void bit_andnot_arr_ffmask(bm::word_t* BMRESTRICT dst,
7277 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7278 {
7279 const bm::word_t* BMRESTRICT src_end = src + bm::set_block_size;
7280 #ifdef BMVECTOPT
7281 VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, ~0u);
7282 #else
7283 bm::wordop_t* dst_ptr = (wordop_t*)dst;
7284 const bm::wordop_t* wrd_ptr = (wordop_t*) src;
7285 const bm::wordop_t* wrd_end = (wordop_t*) src_end;
7286
7287 do
7288 {
7289 dst_ptr[0] = bm::all_bits_mask & ~wrd_ptr[0];
7290 dst_ptr[1] = bm::all_bits_mask & ~wrd_ptr[1];
7291 dst_ptr[2] = bm::all_bits_mask & ~wrd_ptr[2];
7292 dst_ptr[3] = bm::all_bits_mask & ~wrd_ptr[3];
7293 dst_ptr+=4; wrd_ptr+=4;
7294 } while (wrd_ptr < wrd_end);
7295 #endif
7296 }
7297
7298 /*!
7299 \brief bitblock XOR operation.
7300
7301 \param dst - destination block.
7302 \param src - source block.
7303
7304 \returns pointer on destination block.
7305 If returned value equal to src means that block mutation requested.
7306 NULL is valid return value.
7307
7308 @ingroup bitfunc
7309 */
7310 inline
7311 bm::word_t* bit_operation_xor(bm::word_t* BMRESTRICT dst,
7312 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7313 {
7314 BM_ASSERT(dst || src);
7315 if (src == dst) return 0; // XOR rule
7316
7317 bm::word_t* ret = dst;
7318
7319 if (IS_VALID_ADDR(dst)) // The destination block already exists
7320 {
7321 if (!src) return dst;
7322
7323 bit_block_xor(dst, src);
7324 }
7325 else // The destination block does not exist yet
7326 {
7327 if (!src) return dst; // 1 xor 0 = 1
7328
7329 // Here we have two cases:
7330 // if dest block is full or zero if zero we need to copy the source
7331 // otherwise XOR loop against 0xFF...
7332 //BM_ASSERT(dst == 0);
7333 return const_cast<bm::word_t*>(src); // src is the final result
7334 }
7335 return ret;
7336 }
7337
7338 /*!
7339 \brief Performs bitblock XOR operation and calculates bitcount of the result.
7340
7341 \param src1 - bit block start ptr
7342 \param src2 - second bit block
7343
7344 \returns bitcount value
7345
7346 @ingroup bitfunc
7347 */
7348 inline
7349 bm::id_t bit_operation_xor_count(const bm::word_t* BMRESTRICT src1,
7350 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
7351 {
7352 if (src1 == src2)
7353 return 0;
7354
7355 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
7356 {
7357 const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
7358 if (IS_FULL_BLOCK(block))
7359 return bm::gap_max_bits;
7360 return bm::bit_block_count(block);
7361 }
7362 if (src1 == FULL_BLOCK_FAKE_ADDR)
7363 src1 = FULL_BLOCK_REAL_ADDR;
7364 if (src2 == FULL_BLOCK_FAKE_ADDR)
7365 src2 = FULL_BLOCK_REAL_ADDR;
7366
7367 return bit_block_xor_count(src1, src2);
7368 }
7369
7370 /*!
7371 \brief Performs bitblock XOR operation test.
7372
7373 \param src1 - bit block start ptr
7374 \param src2 - second bit block ptr
7375
7376 \returns non zero value if there are bits
7377
7378 @ingroup bitfunc
7379 */
7380 inline
7381 bm::id_t bit_operation_xor_any(const bm::word_t* BMRESTRICT src1,
7382 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
7383 {
7384 if (src1 == src2)
7385 return 0;
7386 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
7387 {
7388 const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
7389 return !bit_is_all_zero(block);
7390 }
7391 return bm::bit_block_xor_any(src1, src2);
7392 }
7393
7394
7395
7396 /**
7397 \brief Inspects block for full zero words
7398
7399 \param blk - bit block pointer
7400 \param data_size - data size
7401
7402 \return size of all non-zero words
7403
7404 @ingroup bitfunc
7405 */
7406 template<class T>
7407 unsigned bit_count_nonzero_size(const T* blk, unsigned data_size) BMNOEXCEPT
7408 {
7409 BM_ASSERT(blk && data_size);
7410 unsigned count = 0;
7411 const T* blk_end = blk + data_size - 2;
7412 do
7413 {
7414 if (*blk == 0)
7415 {
7416 // scan fwd to find 0 island length
7417 const T* blk_j = blk + 1;
7418 for (; blk_j < blk_end; ++blk_j)
7419 {
7420 if (*blk_j != 0)
7421 break;
7422 }
7423 blk = blk_j-1;
7424 count += (unsigned)sizeof(gap_word_t);
7425 }
7426 else
7427 {
7428 // scan fwd to find non-0 island length
7429 const T* blk_j = blk + 1;
7430 for ( ; blk_j < blk_end; ++blk_j)
7431 {
7432 if (*blk_j == 0)
7433 {
7434 // look ahead to identify and ignore short 0-run
7435 if (blk_j[1] | blk_j[2])
7436 {
7437 // skip zero word
7438 ++blk_j;
7439 continue;
7440 }
7441 break;
7442 }
7443 }
7444 count += unsigned(sizeof(gap_word_t));
7445 // count all bit-words now
7446 count += unsigned(blk_j - blk) * unsigned(sizeof(T));
7447 blk = blk_j;
7448 }
7449 ++blk;
7450 }
7451 while(blk < blk_end);
7452 return count + unsigned(2 * sizeof(T));
7453 }
7454
7455
7456 /**
7457 \brief Searches for the next 1 bit in the BIT block
7458 \param block - BIT buffer
7459 \param nbit - bit index to start checking from
7460 \param pos - [out] found value
7461
7462 \return 0 if not found
7463
7464 @ingroup bitfunc
7465 */
7466 inline
7467 unsigned bit_block_find(const bm::word_t* BMRESTRICT block,
7468 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
7469 {
7470 BM_ASSERT(block);
7471 BM_ASSERT(pos);
7472
7473 unsigned nword = unsigned(nbit >> bm::set_word_shift);
7474 unsigned bit_pos = (nbit & bm::set_word_mask);
7475
7476 bm::word_t w = block[nword];
7477 w &= (1u << bit_pos);
7478 if (w)
7479 {
7480 *pos = nbit;
7481 return 1;
7482 }
7483 w = block[nword] >> bit_pos;
7484 w <<= bit_pos; // clear the trailing bits
7485 if (w)
7486 {
7487 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
7488 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t))));
7489 return 1;
7490 }
7491
7492 for (unsigned i = nword+1; i < bm::set_block_size; ++i)
7493 {
7494 w = block[i];
7495 if (w)
7496 {
7497 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
7498 *pos = unsigned(bit_pos + (i * 8u * unsigned(sizeof(bm::word_t))));
7499 return w;
7500 }
7501 } // for i
7502 return 0u;
7503 }
7504
7505
7506
7507
7508 /*!
7509 \brief BIT block find the last set bit (backward search)
7510
7511 \param block - bit block buffer pointer
7512 \param last - index of the last 1 bit (out)
7513 \return true if found
7514
7515 @ingroup bitfunc
7516 */
7517 inline
7518 unsigned bit_find_last(const bm::word_t* BMRESTRICT block,
7519 unsigned* BMRESTRICT last) BMNOEXCEPT
7520 {
7521 BM_ASSERT(block);
7522 BM_ASSERT(last);
7523
7524 // TODO: SIMD version
7525
7526 for (unsigned i = bm::set_block_size-1; true; --i)
7527 {
7528 bm::word_t w = block[i];
7529 if (w)
7530 {
7531 unsigned idx = bm::bit_scan_reverse(w);
7532 *last = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7533 return w;
7534 }
7535 if (i == 0)
7536 break;
7537 } // for i
7538 return 0u;
7539 }
7540
7541 /*!
7542 \brief BIT block find the first set bit
7543
7544 \param block - bit block buffer pointer
7545 \param pos - index of the first 1 bit (out)
7546 \return 0 if not found
7547
7548 @ingroup bitfunc
7549 @internal
7550 */
7551 inline
7552 bool bit_find_first(const bm::word_t* BMRESTRICT block,
7553 unsigned* BMRESTRICT pos) BMNOEXCEPT
7554 {
7555 BM_ASSERT(block);
7556 BM_ASSERT(pos);
7557
7558 #ifdef VECT_BIT_FIND_FIRST
7559 return VECT_BIT_FIND_FIRST(block, pos);
7560 #else
7561 for (unsigned i = 0; i < bm::set_block_size; ++i)
7562 {
7563 bm::word_t w = block[i];
7564 if (w)
7565 {
7566 unsigned idx = bm::bit_scan_forward32(w); // trailing zeros
7567 *pos = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7568 return w;
7569 }
7570 } // for i
7571 return false;
7572 #endif
7573 }
7574
7575 /*!
7576 \brief BIT block find the first set bit
7577
7578 \param block - bit block buffer pointer
7579 \param first - index of the first 1 bit (out)
7580 \param digest - known digest of dst block
7581
7582 \return 0 if not found
7583
7584 @ingroup bitfunc
7585 */
7586 inline
7587 unsigned bit_find_first(const bm::word_t* BMRESTRICT block,
7588 unsigned* BMRESTRICT first,
7589 bm::id64_t digest) BMNOEXCEPT
7590 {
7591 BM_ASSERT(block);
7592 BM_ASSERT(first);
7593 BM_ASSERT(digest);
7594
7595 bm::id64_t t = bm::bmi_blsi_u64(digest); // d & -d;
7596
7597 unsigned wave = bm::word_bitcount64(t - 1);
7598 unsigned off = wave * bm::set_block_digest_wave_size;
7599 for (unsigned i = off; i < bm::set_block_size; ++i)
7600 {
7601 bm::word_t w = block[i];
7602 if (w)
7603 {
7604 unsigned idx = bit_scan_forward32(w); // trailing zeros
7605 *first = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7606 return w;
7607 }
7608 } // for i
7609 return 0u;
7610 }
7611
7612
7613 /*!
7614 \brief BIT block find the first set bit if only 1 bit is set
7615
7616 \param block - bit block buffer pointer
7617 \param first - index of the first 1 bit (out)
7618 \param digest - known digest of dst block
7619
7620 \return 0 if not found
7621
7622 @ingroup bitfunc
7623 */
7624 inline
7625 bool bit_find_first_if_1(const bm::word_t* BMRESTRICT block,
7626 unsigned* BMRESTRICT first,
7627 bm::id64_t digest) BMNOEXCEPT
7628 {
7629 BM_ASSERT(block);
7630 BM_ASSERT(first);
7631
7632 unsigned bc = bm::word_bitcount64(digest);
7633 if (bc != 1)
7634 return false;
7635
7636 bool found = false;
7637 bm::id64_t t = bm::bmi_blsi_u64(digest); // d & -d;
7638
7639 unsigned wave = bm::word_bitcount64(t - 1);
7640 unsigned off = wave * bm::set_block_digest_wave_size;
7641 unsigned i;
7642 for (i = off; i < off + bm::set_block_digest_wave_size; ++i)
7643 {
7644 bm::word_t w = block[i];
7645 if (w)
7646 {
7647 bc = bm::word_bitcount(w);
7648 if (bc != 1)
7649 return false;
7650
7651 unsigned idx = bit_scan_forward32(w); // trailing zeros
7652 *first = unsigned(idx + (i * 8u * sizeof(bm::word_t)));
7653 found = true;
7654 break;
7655 }
7656 } // for i
7657
7658 // check if all other bits are zero
7659 for (++i; i < off + bm::set_block_digest_wave_size; ++i)
7660 {
7661 if (block[i])
7662 return false;
7663 }
7664 return found;
7665 }
7666
7667
7668 /*!
7669 \brief BIT block find position for the rank
7670
7671 \param block - bit block buffer pointer
7672 \param rank - rank to find (must be > 0)
7673 \param nbit_from - start bit position in block
7674 \param nbit_pos - (out)found position
7675
7676 \return 0 if position with rank was found, or
7677 the remaining rank (rank - population count)
7678
7679 @ingroup bitfunc
7680 */
7681 template<typename SIZE_TYPE>
7682 SIZE_TYPE bit_find_rank(const bm::word_t* const block,
7683 SIZE_TYPE rank,
7684 unsigned nbit_from,
7685 unsigned& nbit_pos) BMNOEXCEPT
7686 {
7687 BM_ASSERT(block);
7688 BM_ASSERT(rank);
7689
7690 unsigned nword = nbit_from >> bm::set_word_shift;
7691 BM_ASSERT(nword < bm::set_block_size);
7692
7693 unsigned pos = nbit_from;
7694 bm::id_t nbit = (nbit_from & bm::set_word_mask);
7695
7696 if (nbit)
7697 {
7698 bm::id_t w = block[nword];
7699 w >>= nbit;
7700 unsigned bc = bm::word_bitcount(w);
7701 if (bc < rank) // skip this
7702 {
7703 rank -= bc; pos += unsigned(32u - nbit);
7704 ++nword;
7705 }
7706 else // target word
7707 {
7708 unsigned idx = bm::word_select64(w, unsigned(rank));
7709 nbit_pos = pos + idx;
7710 return 0;
7711 }
7712 }
7713
7714 if (bm::conditional<sizeof(void*) == 8>::test()) // 64-bit fast scan
7715 {
7716 for (; nword < bm::set_block_size-1; nword+=2)
7717 {
7718 bm::id64_t w = (bm::id64_t(block[nword+1]) << 32) | bm::id64_t(block[nword]);
7719 bm::id_t bc = bm::word_bitcount64(w);
7720 if (bc >= rank) // target
7721 {
7722 unsigned idx = bm::word_select64(w, unsigned(rank));
7723 nbit_pos = pos + idx;
7724 return 0;
7725 }
7726 rank -= bc;
7727 pos += 64u;
7728 }
7729 }
7730
7731 for (; nword < bm::set_block_size; ++nword)
7732 {
7733 bm::id_t w = block[nword];
7734 bm::id_t bc = bm::word_bitcount(w);
7735 if (rank > bc)
7736 {
7737 rank -= bc; pos += 32u;
7738 continue;
7739 }
7740 unsigned idx = bm::word_select64(w, unsigned(rank));
7741 nbit_pos = pos + idx;
7742 return 0;
7743 } // for nword
7744 return rank;
7745 }
7746
7747 /**
7748 \brief Find rank in block (GAP or BIT)
7749
7750 \param block - bit block buffer pointer
7751 \param rank - rank to find (must be > 0)
7752 \param nbit_from - start bit position in block
7753 \param nbit_pos - found position
7754
7755 \return 0 if position with rank was found, or
7756 the remaining rank (rank - population count)
7757
7758 @internal
7759 */
7760 template<typename SIZE_TYPE>
7761 SIZE_TYPE block_find_rank(const bm::word_t* const block,
7762 SIZE_TYPE rank,
7763 unsigned nbit_from,
7764 unsigned& nbit_pos) BMNOEXCEPT
7765 {
7766 if (BM_IS_GAP(block))
7767 {
7768 const bm::gap_word_t* const gap_block = BMGAP_PTR(block);
7769 rank = bm::gap_find_rank(gap_block, rank, nbit_from, nbit_pos);
7770 }
7771 else
7772 {
7773 rank = bm::bit_find_rank(block, rank, nbit_from, nbit_pos);
7774 }
7775 return rank;
7776 }
7777
7778
7779
7780 /*!
7781 @brief Choose best representation for a bit-block
7782 @ingroup bitfunc
7783 */
7784 inline
7785 bm::set_representation best_representation(unsigned bit_count,
7786 unsigned total_possible_bitcount,
7787 unsigned gap_count,
7788 unsigned block_size) BMNOEXCEPT
7789 {
7790 unsigned arr_size = unsigned(sizeof(bm::gap_word_t) * bit_count + sizeof(bm::gap_word_t));
7791 unsigned gap_size = unsigned(sizeof(bm::gap_word_t) * gap_count + sizeof(bm::gap_word_t));
7792 unsigned inv_arr_size = unsigned(sizeof(bm::gap_word_t) * (total_possible_bitcount - bit_count) + sizeof(bm::gap_word_t));
7793
7794 if ((gap_size < block_size) && (gap_size < arr_size) && (gap_size < inv_arr_size))
7795 {
7796 return bm::set_gap;
7797 }
7798
7799 if (arr_size < inv_arr_size)
7800 {
7801 if ((arr_size < block_size) && (arr_size < gap_size))
7802 {
7803 return bm::set_array1;
7804 }
7805 }
7806 else
7807 {
7808 if ((inv_arr_size < block_size) && (inv_arr_size < gap_size))
7809 {
7810 return bm::set_array0;
7811 }
7812 }
7813 return bm::set_bitset;
7814 }
7815
7816 /*!
7817 @brief Convert bit block into an array of ints corresponding to 1 bits
7818 @return destination size as a result of block decoding
7819 @ingroup bitfunc
7820 */
7821 template<typename T>
7822 T bit_convert_to_arr(T* BMRESTRICT dest,
7823 const unsigned* BMRESTRICT src,
7824 bm::id_t bits,
7825 unsigned dest_len,
7826 unsigned mask = 0) BMNOEXCEPT
7827 {
7828 T* BMRESTRICT pcurr = dest;
7829 for (unsigned bit_idx=0; bit_idx < bits;
7830 ++src,bit_idx += unsigned(sizeof(*src) * 8))
7831 {
7832 unsigned val = *src ^ mask; // invert value by XOR 0xFF..
7833 if (val == 0)
7834 continue;
7835 if (pcurr + unsigned(sizeof(val)*8) >= dest + dest_len) // insufficient space
7836 return 0;
7837 // popscan loop to decode bits in a word
7838 while (val)
7839 {
7840 bm::id_t t = val & -val;
7841 *pcurr++ = (T)(bm::word_bitcount(t - 1) + bit_idx);
7842 val &= val - 1;
7843 }
7844 } // for
7845 return (T)(pcurr - dest);
7846 }
7847
7848 /**
7849 \brief Checks all conditions and returns true if block consists of only 0 bits
7850 \param blk - Blocks's pointer
7851 \param deep_scan - flag to do full bit block verification (scan)
7852 when deep scan is not requested result can be approximate
7853 \returns true if all bits are in the block are 0
7854
7855 @internal
7856 */
7857 inline
7858 bool check_block_zero(const bm::word_t* blk, bool deep_scan) BMNOEXCEPT
7859 {
7860 if (!blk) return true;
7861 if (IS_FULL_BLOCK(blk)) return false;
7862
7863 bool ret;
7864 if (BM_IS_GAP(blk))
7865 ret = gap_is_all_zero(BMGAP_PTR(blk));
7866 else
7867 ret = deep_scan ? bm::bit_is_all_zero(blk) : 0;
7868 return ret;
7869 }
7870
7871
7872 /**
7873 \brief Checks if block has only 1 bits
7874 \param blk - Block's pointer
7875 \param deep_scan - flag to do full bit block verification (scan)
7876 when deep scan is not requested result can be approximate
7877 \return true if block consists of 1 bits.
7878
7879 @internal
7880 */
7881 inline
7882 bool check_block_one(const bm::word_t* blk, bool deep_scan) BMNOEXCEPT
7883 {
7884 if (blk == 0) return false;
7885
7886 if (BM_IS_GAP(blk))
7887 return bm::gap_is_all_one(BMGAP_PTR(blk));
7888
7889 if (IS_FULL_BLOCK(blk))
7890 return true;
7891
7892 if (!deep_scan)
7893 return false; // block exists - presume it has 0 bits
7894
7895 return bm::is_bits_one((wordop_t*)blk);
7896 }
7897
7898
7899
7900 /*! @brief Calculates memory overhead for number of gap blocks sharing
7901 the same memory allocation table (level lengths table).
7902 @ingroup gapfunc
7903 */
7904 template<typename T>
7905 unsigned gap_overhead(const T* length,
7906 const T* length_end,
7907 const T* glevel_len) BMNOEXCEPT
7908 {
7909 BM_ASSERT(length && length_end && glevel_len);
7910
7911 unsigned overhead = 0;
7912 for (;length < length_end; ++length)
7913 {
7914 unsigned len = *length;
7915 int level = gap_calc_level(len, glevel_len);
7916 BM_ASSERT(level >= 0 && level < (int)bm::gap_levels);
7917 unsigned capacity = glevel_len[level];
7918 BM_ASSERT(capacity >= len);
7919 overhead += capacity - len;
7920 }
7921 return overhead;
7922 }
7923
7924
7925 /*! @brief Finds optimal gap blocks lengths.
7926 @param length - first element of GAP lengths array
7927 @param length_end - end of the GAP lengths array
7928 @param glevel_len - destination GAP lengths array
7929 @ingroup gapfunc
7930 */
7931 template<typename T>
7932 bool improve_gap_levels(const T* length,
7933 const T* length_end,
7934 T* glevel_len) BMNOEXCEPT
7935 {
7936 BM_ASSERT(length && length_end && glevel_len);
7937
7938 size_t lsize = size_t(length_end - length);
7939
7940 BM_ASSERT(lsize);
7941
7942 gap_word_t max_len = 0;
7943 unsigned i;
7944 for (i = 0; i < lsize; ++i)
7945 {
7946 if (length[i] > max_len)
7947 max_len = length[i];
7948 }
7949 if (max_len < 5 || lsize <= bm::gap_levels)
7950 {
7951 glevel_len[0] = T(max_len + 4);
7952 for (i = 1; i < bm::gap_levels; ++i)
7953 {
7954 glevel_len[i] = bm::gap_max_buff_len;
7955 }
7956 return true;
7957 }
7958
7959 glevel_len[bm::gap_levels-1] = T(max_len + 5);
7960
7961 unsigned min_overhead = gap_overhead(length, length_end, glevel_len);
7962 bool is_improved = false;
7963
7964 // main problem solving loop
7965 //
7966 for (i = bm::gap_levels-2; ; --i)
7967 {
7968 unsigned opt_len = 0;
7969 unsigned j;
7970 bool imp_flag = false;
7971 gap_word_t gap_saved_value = glevel_len[i];
7972 for (j = 0; j < lsize; ++j)
7973 {
7974 glevel_len[i] = T(length[j] + 4);
7975 unsigned ov = gap_overhead(length, length_end, glevel_len);
7976 if (ov <= min_overhead)
7977 {
7978 min_overhead = ov;
7979 opt_len = length[j]+4;
7980 imp_flag = true;
7981 }
7982 }
7983 if (imp_flag)
7984 {
7985 glevel_len[i] = (T)opt_len; // length[opt_idx]+4;
7986 is_improved = true;
7987 }
7988 else
7989 {
7990 glevel_len[i] = gap_saved_value;
7991 }
7992 if (i == 0)
7993 break;
7994 }
7995
7996 // Remove duplicates
7997 //
7998 T val = *glevel_len;
7999 T* gp = glevel_len;
8000 T* res = glevel_len;
8001 for (i = 0; i < bm::gap_levels; ++i)
8002 {
8003 if (val != *gp)
8004 {
8005 val = *gp;
8006 *++res = val;
8007 }
8008 ++gp;
8009 }
8010
8011 // Filling the "unused" part with max. possible value
8012 while (++res < (glevel_len + bm::gap_levels))
8013 {
8014 *res = bm::gap_max_buff_len;
8015 }
8016
8017 return is_improved;
8018
8019 }
8020
8021 /*!
8022 \brief Find first bit which is different between two blocks (GAP or bit)
8023 \param blk - block 1
8024 \param arg_blk - block 2
8025 \param pos - out - position of difference (undefined if blocks are equal)
8026 \return true if difference was found
8027 @internal
8028 */
8029 inline
8030 bool block_find_first_diff(const bm::word_t* BMRESTRICT blk,
8031 const bm::word_t* BMRESTRICT arg_blk,
8032 unsigned* BMRESTRICT pos) BMNOEXCEPT
8033 {
8034 // If one block is zero we check if the other one has at least
8035 // one bit ON
8036
8037 if (!blk || !arg_blk)
8038 {
8039 const bm::word_t* pblk; bool is_gap;
8040 if (blk)
8041 {
8042 pblk = blk;
8043 is_gap = BM_IS_GAP(blk);
8044 }
8045 else
8046 {
8047 pblk = arg_blk;
8048 is_gap = BM_IS_GAP(arg_blk);
8049 }
8050
8051 if (is_gap)
8052 {
8053 unsigned found = bm::gap_find_first(BMGAP_PTR(pblk), pos);
8054 if (found)
8055 return true;
8056 }
8057 else
8058 {
8059 unsigned found = bm::bit_find_first(pblk, pos);
8060 if (found)
8061 return true;
8062 }
8063 return false;
8064 }
8065
8066 bool arg_gap = BM_IS_GAP(arg_blk);
8067 bool gap = BM_IS_GAP(blk);
8068
8069 if (arg_gap != gap)
8070 {
8071 //BM_DECLARE_TEMP_BLOCK(temp_blk);
8072 bm::bit_block_t temp_blk;
8073 bm::word_t* blk1; bm::word_t* blk2;
8074
8075 if (gap)
8076 {
8077 bm::gap_convert_to_bitset((bm::word_t*)temp_blk,
8078 BMGAP_PTR(blk));
8079 blk1 = (bm::word_t*)temp_blk;
8080 blk2 = (bm::word_t*)arg_blk;
8081 }
8082 else
8083 {
8084 bm::gap_convert_to_bitset((bm::word_t*)temp_blk,
8085 BMGAP_PTR(arg_blk));
8086 blk1 = (bm::word_t*)blk;
8087 blk2 = (bm::word_t*)temp_blk;
8088 }
8089 bool found = bm::bit_find_first_diff(blk1, blk2, pos);
8090 if (found)
8091 return true;
8092 }
8093 else
8094 {
8095 if (gap)
8096 {
8097 bool found =
8098 bm::gap_find_first_diff(BMGAP_PTR(blk),
8099 BMGAP_PTR(arg_blk), pos);
8100 if (found)
8101 return true;
8102 }
8103 else
8104 {
8105 bool found = bm::bit_find_first_diff(blk, arg_blk, pos);
8106 if (found)
8107 return true;
8108 }
8109 }
8110 return false;
8111 }
8112
8113
8114
8115
8116 /**
8117 Bit-block get adapter, takes bitblock and represents it as a
8118 get_32() accessor function
8119 \internal
8120 */
8121 class bitblock_get_adapter
8122 {
8123 public:
8124 bitblock_get_adapter(const bm::word_t* bit_block) : b_(bit_block) {}
8125
8126 BMFORCEINLINE
8127 bm::word_t get_32() BMNOEXCEPT { return *b_++; }
8128 private:
8129 const bm::word_t* b_;
8130 };
8131
8132
8133 /**
8134 Bit-block store adapter, takes bitblock and saves results into it
8135 \internal
8136 */
8137 class bitblock_store_adapter
8138 {
8139 public:
8140 bitblock_store_adapter(bm::word_t* bit_block) : b_(bit_block) {}
8141 BMFORCEINLINE
8142 void push_back(bm::word_t w) { *b_++ = w; }
8143 private:
8144 bm::word_t* b_;
8145 };
8146
8147 /**
8148 Bit-block sum adapter, takes values and sums it
8149 /internal
8150 */
8151 class bitblock_sum_adapter
8152 {
8153 public:
8154 bitblock_sum_adapter() : sum_(0) {}
8155 BMFORCEINLINE
8156 void push_back(bm::word_t w) BMNOEXCEPT { this->sum_+= w; }
8157 /// Get accumulated sum
8158 bm::word_t sum() const BMNOEXCEPT { return this->sum_; }
8159 private:
8160 bm::word_t sum_;
8161 };
8162
8163 /**
8164 Adapter to get words from a range stream
8165 (see range serialized bit-block)
8166 \internal
8167 */
8168 template<class DEC> class decoder_range_adapter
8169 {
8170 public:
8171 decoder_range_adapter(DEC& dec, unsigned from_idx, unsigned to_idx)
8172 : decoder_(dec),
8173 from_(from_idx),
8174 to_(to_idx),
8175 cnt_(0)
8176 {}
8177
8178 bm::word_t get_32() BMNOEXCEPT
8179 {
8180 if (cnt_ < from_ || cnt_ > to_)
8181 {
8182 ++cnt_; return 0;
8183 }
8184 ++cnt_;
8185 return decoder_.get_32();
8186 }
8187
8188 private:
8189 DEC& decoder_;
8190 unsigned from_;
8191 unsigned to_;
8192 unsigned cnt_;
8193 };
8194
8195
8196 /*!
8197 Abstract recombination algorithm for two bit-blocks
8198 Bit blocks can come as dserialization decoders or bit-streams
8199 */
8200 template<class It1, class It2, class BinaryOp, class Encoder>
8201 void bit_recomb(It1& it1, It2& it2,
8202 BinaryOp& op,
8203 Encoder& enc,
8204 unsigned block_size = bm::set_block_size) BMNOEXCEPT
8205 {
8206 for (unsigned i = 0; i < block_size; ++i)
8207 {
8208 bm::word_t w1 = it1.get_32();
8209 bm::word_t w2 = it2.get_32();
8210 bm::word_t w = op(w1, w2);
8211 enc.push_back( w );
8212 } // for
8213 }
8214
8215 /// Bit AND functor
8216 template<typename W> struct bit_AND
8217 {
8218 W operator()(W w1, W w2) BMNOEXCEPT { return w1 & w2; }
8219 };
8220
8221 /// Bit OR functor
8222 template<typename W> struct bit_OR
8223 {
8224 W operator()(W w1, W w2) BMNOEXCEPT { return w1 | w2; }
8225 };
8226
8227 /// Bit SUB functor
8228 template<typename W> struct bit_SUB
8229 {
8230 W operator()(W w1, W w2) BMNOEXCEPT { return w1 & ~w2; }
8231 };
8232
8233 /// Bit XOR functor
8234 template<typename W> struct bit_XOR
8235 {
8236 W operator()(W w1, W w2) BMNOEXCEPT { return w1 ^ w2; }
8237 };
8238
8239 /// Bit ASSIGN functor
8240 template<typename W> struct bit_ASSIGN
8241 {
8242 W operator()(W, W w2) BMNOEXCEPT { return w2; }
8243 };
8244
8245 /// Bit COUNT functor
8246 template<typename W> struct bit_COUNT
8247 {
8248 W operator()(W w1, W w2) BMNOEXCEPT
8249 {
8250 w1 = 0;
8251 BM_INCWORD_BITCOUNT(w1, w2);
8252 return w1;
8253 }
8254 };
8255
8256 /// Bit COUNT AND functor
8257 template<typename W> struct bit_COUNT_AND
8258 {
8259 W operator()(W w1, W w2) BMNOEXCEPT
8260 {
8261 W r = 0;
8262 BM_INCWORD_BITCOUNT(r, w1 & w2);
8263 return r;
8264 }
8265 };
8266
8267 /// Bit COUNT XOR functor
8268 template<typename W> struct bit_COUNT_XOR
8269 {
8270 W operator()(W w1, W w2) BMNOEXCEPT
8271 {
8272 W r = 0;
8273 BM_INCWORD_BITCOUNT(r, w1 ^ w2);
8274 return r;
8275 }
8276 };
8277
8278 /// Bit COUNT OR functor
8279 template<typename W> struct bit_COUNT_OR
8280 {
8281 W operator()(W w1, W w2) BMNOEXCEPT
8282 {
8283 W r = 0;
8284 BM_INCWORD_BITCOUNT(r, w1 | w2);
8285 return r;
8286 }
8287 };
8288
8289
8290 /// Bit COUNT SUB AB functor
8291 template<typename W> struct bit_COUNT_SUB_AB
8292 {
8293 W operator()(W w1, W w2) BMNOEXCEPT
8294 {
8295 W r = 0;
8296 BM_INCWORD_BITCOUNT(r, w1 & (~w2));
8297 return r;
8298 }
8299 };
8300
8301 /// Bit SUB BA functor
8302 template<typename W> struct bit_COUNT_SUB_BA
8303 {
8304 W operator()(W w1, W w2) BMNOEXCEPT
8305 {
8306 W r = 0;
8307 BM_INCWORD_BITCOUNT(r, w2 & (~w1));
8308 return r;
8309 }
8310 };
8311
8312 /// Bit COUNT A functor
8313 template<typename W> struct bit_COUNT_A
8314 {
8315 W operator()(W w1, W ) BMNOEXCEPT
8316 {
8317 W r = 0;
8318 BM_INCWORD_BITCOUNT(r, w1);
8319 return r;
8320 }
8321 };
8322
8323 /// Bit COUNT B functor
8324 template<typename W> struct bit_COUNT_B
8325 {
8326 W operator()(W, W w2) BMNOEXCEPT
8327 {
8328 W r = 0;
8329 BM_INCWORD_BITCOUNT(r, w2);
8330 return r;
8331 }
8332 };
8333
8334 typedef
8335 void (*gap_operation_to_bitset_func_type)(unsigned*,
8336 const gap_word_t*);
8337
8338 typedef
8339 gap_word_t* (*gap_operation_func_type)(const gap_word_t* BMRESTRICT,
8340 const gap_word_t* BMRESTRICT,
8341 gap_word_t* BMRESTRICT,
8342 unsigned& );
8343
8344 typedef
8345 bm::id_t (*bit_operation_count_func_type)(const bm::word_t* BMRESTRICT,
8346 const bm::word_t* BMRESTRICT);
8347
8348
8349 template<bool T>
8350 struct operation_functions
8351 {
8352 static
8353 gap_operation_to_bitset_func_type gap2bit_table_[bm::set_END];
8354 static
8355 gap_operation_func_type gapop_table_[bm::set_END];
8356 static
8357 bit_operation_count_func_type bit_op_count_table_[bm::set_END];
8358
8359 static
8360 gap_operation_to_bitset_func_type gap_op_to_bit(unsigned i)
8361 {
8362 return gap2bit_table_[i];
8363 }
8364
8365 static
8366 gap_operation_func_type gap_operation(unsigned i)
8367 {
8368 return gapop_table_[i];
8369 }
8370
8371 static
8372 bit_operation_count_func_type bit_operation_count(unsigned i)
8373 {
8374 return bit_op_count_table_[i];
8375 }
8376 };
8377
8378 template<bool T>
8379 gap_operation_to_bitset_func_type
8380 operation_functions<T>::gap2bit_table_[bm::set_END] = {
8381 &gap_and_to_bitset<bm::gap_word_t>, // set_AND
8382 &gap_add_to_bitset<bm::gap_word_t>, // set_OR
8383 &gap_sub_to_bitset<bm::gap_word_t>, // set_SUB
8384 &gap_xor_to_bitset<bm::gap_word_t>, // set_XOR
8385 0
8386 };
8387
8388 template<bool T>
8389 gap_operation_func_type
8390 operation_functions<T>::gapop_table_[bm::set_END] = {
8391 &gap_operation_and, // set_AND
8392 &gap_operation_or, // set_OR
8393 &gap_operation_sub, // set_SUB
8394 &gap_operation_xor, // set_XOR
8395 0
8396 };
8397
8398
8399 template<bool T>
8400 bit_operation_count_func_type
8401 operation_functions<T>::bit_op_count_table_[bm::set_END] = {
8402 0, // set_AND
8403 0, // set_OR
8404 0, // set_SUB
8405 0, // set_XOR
8406 0, // set_ASSIGN
8407 0, // set_COUNT
8408 &bit_operation_and_count, // set_COUNT_AND
8409 &bit_operation_xor_count, // set_COUNT_XOR
8410 &bit_operation_or_count, // set_COUNT_OR
8411 &bit_operation_sub_count, // set_COUNT_SUB_AB
8412 &bit_operation_sub_count_inv, // set_COUNT_SUB_BA
8413 0, // set_COUNT_A
8414 0, // set_COUNT_B
8415 };
8416
8417 /**
8418 Size of bit decode wave in words
8419 @internal
8420 */
8421 const unsigned short set_bitscan_wave_size = 4;
8422 /*!
8423 \brief Unpacks word wave (Nx 32-bit words)
8424 \param w_ptr - pointer on wave start
8425 \param bits - pointer on the result array
8426 \return number of bits in the list
8427
8428 @ingroup bitfunc
8429 @internal
8430 */
8431 inline
8432 unsigned short
8433 bitscan_wave(const bm::word_t* BMRESTRICT w_ptr,
8434 unsigned char* BMRESTRICT bits) BMNOEXCEPT
8435 {
8436 bm::word_t w0, w1;
8437 unsigned short cnt0;
8438
8439 w0 = w_ptr[0];
8440 w1 = w_ptr[1];
8441
8442 #if defined(BMAVX512OPT) || defined(BMAVX2OPT) || defined(BMSSE42OPT)
8443 // combine into 64-bit word and scan (when HW popcnt64 is available)
8444 bm::id64_t w = (bm::id64_t(w1) << 32) | w0;
8445 cnt0 = (unsigned short) bm::bitscan_popcnt64(w, bits);
8446
8447 w0 = w_ptr[2];
8448 w1 = w_ptr[3];
8449 w = (bm::id64_t(w1) << 32) | w0;
8450 cnt0 += bm::bitscan_popcnt64(w, bits + cnt0, 64);
8451 #else
8452 // decode wave as two 32-bit bitscan decodes
8453 cnt0 = bm::bitscan_popcnt(w0, bits);
8454 cnt0 += bm::bitscan_popcnt(w1, bits + cnt0, 32);
8455
8456 w0 = w_ptr[2];
8457 w1 = w_ptr[3];
8458 cnt0 += bm::bitscan_popcnt(w0, bits + cnt0, 64);
8459 cnt0 += bm::bitscan_popcnt(w1, bits + cnt0, 64+32);
8460 #endif
8461 return cnt0;
8462 }
8463
8464 #if defined (BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
8465 /**
8466 bit index to word gather-scatter algorithm (SIMD)
8467 @ingroup bitfunc
8468 @internal
8469 */
8470 inline
8471 void bit_block_gather_scatter(unsigned* BMRESTRICT arr,
8472 const bm::word_t* BMRESTRICT blk,
8473 const unsigned* BMRESTRICT idx,
8474 unsigned size, unsigned start,
8475 unsigned bit_idx) BMNOEXCEPT
8476 {
8477 typedef unsigned TRGW;
8478 typedef unsigned IDX;
8479 #if defined(BM64_SSE4)
8480 // optimized for unsigned
8481 if (bm::conditional<sizeof(TRGW)==4 && sizeof(IDX)==4>::test())
8482 {
8483 sse4_bit_block_gather_scatter(arr, blk, idx, size, start, bit_idx);
8484 return;
8485 }
8486 #elif defined(BM64_AVX2) || defined(BM64_AVX512)
8487 if (bm::conditional<sizeof(TRGW)==4 && sizeof(IDX)==4>::test())
8488 {
8489 avx2_bit_block_gather_scatter(arr, blk, idx, size, start, bit_idx);
8490 return;
8491 }
8492 #else
8493 BM_ASSERT(0);
8494 #endif
8495 }
8496 #endif
8497
8498 /**
8499 bit index to word gather-scatter algorithm
8500 @ingroup bitfunc
8501 @internal
8502 */
8503 template<typename TRGW, typename IDX, typename SZ>
8504 void bit_block_gather_scatter(TRGW* BMRESTRICT arr,
8505 const bm::word_t* BMRESTRICT blk,
8506 const IDX* BMRESTRICT idx,
8507 SZ size, SZ start, unsigned bit_idx) BMNOEXCEPT
8508 {
8509 // TODO: SIMD for 64-bit index sizes and 64-bit target value size
8510 //
8511 TRGW mask1 = 1;
8512 const SZ len = (size - start);
8513 const SZ len_unr = len - (len % 2);
8514 SZ k = 0;
8515 for (; k < len_unr; k+=2)
8516 {
8517 const SZ base = start + k;
8518 const unsigned nbitA = unsigned(idx[base] & bm::set_block_mask);
8519 arr[base] |= (TRGW(bool(blk[nbitA >> bm::set_word_shift] &
8520 (mask1 << (nbitA & bm::set_word_mask)))) << bit_idx);
8521 const unsigned nbitB = unsigned(idx[base + 1] & bm::set_block_mask);
8522 arr[base+1] |= (TRGW(bool(blk[nbitB >> bm::set_word_shift] &
8523 (mask1 << (nbitB & bm::set_word_mask)))) << bit_idx);
8524 } // for k
8525 for (; k < len; ++k)
8526 {
8527 unsigned nbit = unsigned(idx[start + k] & bm::set_block_mask);
8528 arr[start + k] |= (TRGW(bool(blk[nbit >> bm::set_word_shift] &
8529 (mask1 << (nbit & bm::set_word_mask)))) << bit_idx);
8530 } // for k
8531 }
8532
8533 /**
8534 @brief block boundaries look ahead U32
8535
8536 @param idx - array to look into
8537 @param size - array size
8538 @param nb - block number to look ahead
8539 @param start - start offset in idx
8540
8541 @return block boundary offset end (no more match at the returned offset)
8542
8543 @internal
8544 */
8545 inline
8546 bm::id64_t idx_arr_block_lookup_u64(const bm::id64_t* idx,
8547 bm::id64_t size, bm::id64_t nb, bm::id64_t start) BMNOEXCEPT
8548 {
8549 BM_ASSERT(idx);
8550 BM_ASSERT(start < size);
8551
8552 // TODO: SIMD for 64-bit index vector
8553 for (;(start < size) &&
8554 (nb == (idx[start] >> bm::set_block_shift)); ++start)
8555 {}
8556 return start;
8557 }
8558
8559 /**
8560 @brief block boundaries look ahead U32
8561
8562 @param idx - array to look into
8563 @param size - array size
8564 @param nb - block number to look ahead
8565 @param start - start offset in idx
8566
8567 @return block boundary offset end (no more match at the returned offset)
8568
8569 @internal
8570 */
8571 inline
8572 unsigned idx_arr_block_lookup_u32(const unsigned* idx,
8573 unsigned size, unsigned nb, unsigned start) BMNOEXCEPT
8574 {
8575 BM_ASSERT(idx);
8576 BM_ASSERT(start < size);
8577
8578 #if defined(VECT_ARR_BLOCK_LOOKUP)
8579 return VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start);
8580 #else
8581 for (;(start < size) &&
8582 (nb == unsigned(idx[start] >> bm::set_block_shift)); ++start)
8583 {}
8584 return start;
8585 #endif
8586 }
8587
8588 // --------------------------------------------------------------
8589
8590
8591 /**
8592 @brief set bits in a bit-block using global index
8593
8594 @param idx - array to look into
8595 @param block - block pointer to set bits
8596 @param start - index array start
8597 @param stop - index array stop in a range [start..stop)
8598
8599 @return block boundary offset end (no more match at the returned offset)
8600
8601 @internal
8602 @ingroup bitfunc
8603 */
8604 inline
8605 void set_block_bits_u64(bm::word_t* BMRESTRICT block,
8606 const bm::id64_t* BMRESTRICT idx,
8607 bm::id64_t start, bm::id64_t stop) BMNOEXCEPT
8608 {
8609 // TODO: SIMD for 64-bit mode
8610 for (bm::id64_t i = start; i < stop; ++i)
8611 {
8612 bm::id64_t n = idx[i];
8613 unsigned nbit = unsigned(n & bm::set_block_mask);
8614 unsigned nword = nbit >> bm::set_word_shift;
8615 nbit &= bm::set_word_mask;
8616 block[nword] |= (1u << nbit);
8617 } // for i
8618 }
8619
8620
8621 /**
8622 @brief set bits in a bit-block using global index
8623
8624 @param idx - array to look into
8625 @param block - block pointer to set bits
8626 @param start - index array start
8627 @param stop - index array stop in a range [start..stop)
8628
8629 @return block boundary offset end (no more match at the returned offset)
8630
8631 @internal
8632 @ingroup bitfunc
8633 */
8634 inline
8635 void set_block_bits_u32(bm::word_t* BMRESTRICT block,
8636 const unsigned* BMRESTRICT idx,
8637 unsigned start, unsigned stop ) BMNOEXCEPT
8638 {
8639 #if defined(VECT_SET_BLOCK_BITS)
8640 VECT_SET_BLOCK_BITS(block, idx, start, stop);
8641 #else
8642 for (unsigned i = start; i < stop; ++i)
8643 {
8644 unsigned n = idx[i];
8645 unsigned nbit = unsigned(n & bm::set_block_mask);
8646 unsigned nword = nbit >> bm::set_word_shift;
8647 nbit &= bm::set_word_mask;
8648 block[nword] |= (1u << nbit);
8649 } // for i
8650 #endif
8651 }
8652
8653
8654
8655 // --------------------------------------------------------------
8656
8657 /**
8658 @brief array range detector
8659 @internal
8660 */
8661 inline
8662 bool block_ptr_array_range(bm::word_t** arr,
8663 unsigned& left, unsigned& right) BMNOEXCEPT
8664 {
8665 BM_ASSERT(arr);
8666
8667 unsigned i, j;
8668 for (i = 0; i < bm::set_sub_array_size; ++i)
8669 {
8670 if (arr[i])
8671 {
8672 left = i;
8673 break;
8674 }
8675 }
8676 if (i == bm::set_sub_array_size)
8677 {
8678 left = right = 0;
8679 return false; // nothing here
8680 }
8681 for (j = bm::set_sub_array_size-1; j != i; --j)
8682 {
8683 if (arr[j])
8684 break;
8685 }
8686 right = j;
8687 return true;
8688 }
8689
8690 // --------------------------------------------------------------
8691
8692 /**
8693 Linear lower bound search in unsigned array
8694 @internal
8695 */
8696 inline
8697 unsigned lower_bound_linear_u32(const unsigned* arr, unsigned target,
8698 unsigned from, unsigned to) BMNOEXCEPT
8699 {
8700 BM_ASSERT(arr);
8701 BM_ASSERT(from <= to);
8702
8703 #if defined(VECT_LOWER_BOUND_SCAN_U32)
8704 return VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to);
8705 #else
8706 for (; from <= to; ++from)
8707 {
8708 if (arr[from] >= target)
8709 break;
8710 }
8711 return from;
8712 #endif
8713 }
8714
8715 /**
8716 Linear lower bound search in unsigned LONG array
8717 @internal
8718 */
8719 inline
8720 unsigned lower_bound_linear_u64(const unsigned long long* arr,
8721 unsigned long long target,
8722 unsigned from, unsigned to) BMNOEXCEPT
8723 {
8724 BM_ASSERT(arr);
8725 BM_ASSERT(from <= to);
8726
8727 // TODO: implement vectorized scan on u64 ints
8728 for (; from <= to; ++from)
8729 {
8730 if (arr[from] >= target)
8731 break;
8732 }
8733 return from;
8734 }
8735
8736
8737
8738 // --------------------------------------------------------------
8739
8740 /**
8741 Hybrid, binary-linear lower bound search in unsigned array
8742 @internal
8743 */
8744 inline
8745 unsigned lower_bound_u32(const unsigned* arr, unsigned target,
8746 unsigned from, unsigned to) BMNOEXCEPT
8747 {
8748 BM_ASSERT(arr);
8749 BM_ASSERT(from <= to);
8750 const unsigned linear_cutoff = 32;
8751
8752 unsigned l = from; unsigned r = to;
8753 unsigned dist = r - l;
8754 if (dist < linear_cutoff)
8755 {
8756 return bm::lower_bound_linear_u32(arr, target, l, r);
8757 }
8758
8759 while (l <= r)
8760 {
8761 unsigned mid = (r-l)/2+l;
8762 if (arr[mid] < target)
8763 l = mid+1;
8764 else
8765 r = mid-1;
8766 dist = r - l;
8767 if (dist < linear_cutoff)
8768 {
8769 return bm::lower_bound_linear_u32(arr, target, l, r);
8770 }
8771 }
8772 return l;
8773 }
8774
8775 /**
8776 Hybrid, binary-linear lower bound search in unsigned LONG array
8777 @internal
8778 */
8779 inline
8780 unsigned lower_bound_u64(const unsigned long long* arr,
8781 unsigned long long target,
8782 unsigned from, unsigned to) BMNOEXCEPT
8783 {
8784 BM_ASSERT(arr);
8785 BM_ASSERT(from <= to);
8786 const unsigned linear_cutoff = 32;
8787
8788 unsigned l = from; unsigned r = to;
8789 unsigned dist = r - l;
8790 if (dist < linear_cutoff)
8791 {
8792 return bm::lower_bound_linear_u64(arr, target, l, r);
8793 }
8794
8795 while (l <= r)
8796 {
8797 unsigned mid = (r - l) / 2 + l;
8798 if (arr[mid] < target)
8799 l = mid + 1;
8800 else
8801 r = mid - 1;
8802 dist = r - l;
8803 if (dist < linear_cutoff)
8804 {
8805 return bm::lower_bound_linear_u64(arr, target, l, r);
8806 }
8807 }
8808 return l;
8809 }
8810
8811
8812 /**
8813 calculate bvector<> global bit-index from block-local coords
8814 @return bit index in linear bit-vector coordinates
8815 @internal
8816 */
8817 #ifdef BM64ADDR
8818 inline
8819 bm::id64_t block_to_global_index(unsigned i, unsigned j,
8820 unsigned block_idx) BMNOEXCEPT
8821 {
8822 bm::id64_t base_idx = bm::id64_t(i) * bm::set_sub_array_size * bm::gap_max_bits;
8823 base_idx += j * bm::gap_max_bits;
8824 return block_idx + base_idx;
8825 }
8826 #else
8827 inline
8828 bm::id_t block_to_global_index(unsigned i, unsigned j,
8829 unsigned block_idx) BMNOEXCEPT
8830 {
8831 unsigned base_idx = i * bm::set_sub_array_size * bm::gap_max_bits;
8832 base_idx += j * bm::gap_max_bits;
8833 return block_idx + base_idx;
8834 }
8835 #endif
8836
8837 // --------------------------------------------------------------
8838 // Functions to work with int values stored in 64-bit pointers
8839 // --------------------------------------------------------------
8840
8841 /*!
8842 \brief helper union to interpret pointer as integers
8843 @internal
8844 */
8845 union ptr_payload_t
8846 {
8847 bm::word_t* blk;
8848 bm::id64_t i64;
8849 unsigned short i16[4];
8850 };
8851
8852 /*!
8853 Test presense of value in payload pointer
8854 @internal
8855 */
8856 inline
8857 bm::id64_t ptrp_test(ptr_payload_t ptr, bm::gap_word_t v) BMNOEXCEPT
8858 {
8859 if (v == 0)
8860 {
8861 return (ptr.i16[1] == 0);
8862 }
8863 bm::id64_t r = (ptr.i16[1] == v) | (ptr.i16[2] == v) | (ptr.i16[3] == v);
8864 return r;
8865 }
8866
8867 // --------------------------------------------------------------------------
8868
8869 } // namespace bm
8870
8871 #endif
bm::gap_shift_l1
bool gap_shift_l1(T *BMRESTRICT buf, unsigned co_flag, unsigned *BMRESTRICT new_len) BMNOEXCEPT
Left shift GAP block by 1 bit.
Definition: bmfunc.h:2946
bm::gap_find_interval_start
bool gap_find_interval_start(const T *const BMRESTRICT buf, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the first 1 bit in the 111 interval of a GAP block.
Definition: bmfunc.h:2016
bm::bit_andnot_arr_ffmask
void bit_andnot_arr_ffmask(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock AND NOT with constant ~0 mask operation.
Definition: bmfunc.h:7276
bm::operation_functions::gap_operation
static gap_operation_func_type gap_operation(unsigned i)
Definition: bmfunc.h:8366
BM_VECT_ALIGN
#define BM_VECT_ALIGN
Definition: bmdef.h:359
bm::block_find_interval_start
unsigned block_find_interval_start(const bm::word_t *BMRESTRICT block, unsigned nbit_from, unsigned *BMRESTRICT found_nbit) BMNOEXCEPT
Find start of the current 111 interval.
Definition: bmfunc.h:5528
bm::find_not_null_ptr
bool find_not_null_ptr(bm::word_t ***arr, N start, N size, N *pos) BMNOEXCEPT
Definition: bmfunc.h:923
bm::bit_block_change_bc
void bit_block_change_bc(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT gc, unsigned *BMRESTRICT bc) BMNOEXCEPT
Definition: bmfunc.h:4581
bm::bmi_blsi_u64
BMFORCEINLINE unsigned long long bmi_blsi_u64(unsigned long long w)
Definition: bmutil.h:342
VECT_BITCOUNT_OR
#define VECT_BITCOUNT_OR(first, last, mask)
Definition: bmsse4.h:1725
bm::gap_count_or
BMFORCEINLINE unsigned gap_count_or(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount OR operation test.
Definition: bmfunc.h:5810
bm::bit_block_sub_count
unsigned bit_block_sub_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function SUBs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition: bmfunc.h:6319
bm::bit_block_sub
bm::id64_t bit_block_sub(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock SUB (AND NOT) operation. Function does not analyse availability of source and destinat...
Definition: bmfunc.h:7019
bm::bit_block_xor_2way
bm::id64_t bit_block_xor_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
2 way (target := source1 ^ source2) bitblock XOR operation.
Definition: bmfunc.h:6825
bm::bit_is_all_zero
bool bit_is_all_zero(const bm::word_t *BMRESTRICT start) BMNOEXCEPT
Returns "true" if all bits in the block are 0.
Definition: bmfunc.h:1045
bm::sse42_gap_test
unsigned sse42_gap_test(const unsigned short *BMRESTRICT buf, unsigned pos)
Hybrid binary search, starts as binary, then switches to scan.
Definition: bmsse4.h:1106
VECT_BLOCK_CHANGE_BC
#define VECT_BLOCK_CHANGE_BC(block, gc, bc)
Definition: bmsse4.h:1822
bm::bit_block_t::bunion_t
Definition: bmutil.h:58
IS_EMPTY_BLOCK
#define IS_EMPTY_BLOCK(addr)
Definition: bmdef.h:154
bm::bit_block_count
bm::id_t bit_block_count(const bm::word_t *block) BMNOEXCEPT
Bitcount for bit block.
Definition: bmfunc.h:4379
bm::d_copy_func
d-Gap copy functor
Definition: bmfunc.h:2166
bm::bv_statistics::max_serialize_mem
size_t max_serialize_mem
estimated maximum memory for serialization
Definition: bmfunc.h:60
bm::bit_decode_cache
bit-decode cache structure
Definition: bmfunc.h:132
bm::bv_statistics::gaps_by_level
unsigned long long gaps_by_level[bm::gap_levels]
number of GAP blocks at each level
Definition: bmfunc.h:64
bm::gap_operation_sub
gap_word_t * gap_operation_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP SUB (AND NOT) operation.
Definition: bmfunc.h:5835
bm::gap_bitset_or_count
bm::id_t gap_bitset_or_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block OR masked by GAP block.
Definition: bmfunc.h:3790
BM_IS_GAP
#define BM_IS_GAP(ptr)
Definition: bmdef.h:181
bm::block_to_global_index
bm::id_t block_to_global_index(unsigned i, unsigned j, unsigned block_idx) BMNOEXCEPT
calculate bvector<> global bit-index from block-local coords
Definition: bmfunc.h:8828
bm::gapcmp
int gapcmp(const T *buf1, const T *buf2) BMNOEXCEPT
Lexicographical comparison of GAP buffers.
Definition: bmfunc.h:2256
bm::copy_to_array_functor::copy_to_array_functor
copy_to_array_functor(B *bits)
Definition: bmfunc.h:389
bm::lower_bound_linear_u32
unsigned lower_bound_linear_u32(const unsigned *arr, unsigned target, unsigned from, unsigned to) BMNOEXCEPT
Linear lower bound search in unsigned array.
Definition: bmfunc.h:8697
bm::bit_operation_sub_count_inv
bm::id_t bit_operation_sub_count_inv(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs inverted bitblock SUB operation and calculates bitcount of the result.
Definition: bmfunc.h:6626
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx) BMNOEXCEPT
Definition: bmfunc.h:394
bm::set_block_bits_u32
void set_block_bits_u32(bm::word_t *BMRESTRICT block, const unsigned *BMRESTRICT idx, unsigned start, unsigned stop) BMNOEXCEPT
set bits in a bit-block using global index
Definition: bmfunc.h:8635
bm::set_block_size
const unsigned set_block_size
Definition: bmconst.h:54
bm::gap_operation_func_type
gap_word_t *(* gap_operation_func_type)(const gap_word_t *BMRESTRICT, const gap_word_t *BMRESTRICT, gap_word_t *BMRESTRICT, unsigned &)
Definition: bmfunc.h:8339
bm::gap_control_sum
unsigned gap_control_sum(const T *buf) BMNOEXCEPT
Calculates sum of all words in GAP block. (For debugging purposes)
Definition: bmfunc.h:3902
bm::gap_any_range
bool gap_any_range(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if any bits are 1 in GAP buffer in the [left, right] range.
Definition: bmfunc.h:1933
bm::pair::first
First first
Definition: bmfunc.h:123
bm::ptr_payload_t
helper union to interpret pointer as integers
Definition: bmfunc.h:8845
bm::bit_block_find_interval_end
bool bit_block_find_interval_end(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the last 1 bit in the 111 interval of a BIT block.
Definition: bmfunc.h:5373
bm::gap_xor_to_bitset
void gap_xor_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
XOR GAP block to bitblock.
Definition: bmfunc.h:3408
bm::set_gap_level
void set_gap_level(T *buf, int level) BMNOEXCEPT
Sets GAP block capacity level.
Definition: bmfunc.h:4020
bm::bitblock_sum_adapter::bitblock_sum_adapter
bitblock_sum_adapter()
Definition: bmfunc.h:8154
bm::count_trailing_zeros_u64
unsigned count_trailing_zeros_u64(bm::id64_t w) BMNOEXCEPT
64-bit bit-scan fwd
Definition: bmutil.h:381
bm::bit_find_first
bool bit_find_first(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT pos) BMNOEXCEPT
BIT block find the first set bit.
Definition: bmfunc.h:7552
bm::bit_scan_forward32
unsigned bit_scan_forward32(unsigned value) BMNOEXCEPT
Definition: bmutil.h:316
VECT_SUB_BLOCK
#define VECT_SUB_BLOCK(dst, src)
Definition: bmsse4.h:1761
bm::bit_operation_xor_any
bm::id_t bit_operation_xor_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock XOR operation test.
Definition: bmfunc.h:7381
bm::set_AND
@ set_AND
Definition: bmconst.h:154
bm::bits_in_block
const unsigned bits_in_block
Definition: bmconst.h:113
bm::bit_COUNT::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8248
bm::gap_count_xor
BMFORCEINLINE unsigned gap_count_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount XOR operation test.
Definition: bmfunc.h:5765
bm::bit_operation_and_count
bm::id_t bit_operation_and_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock AND operation and calculates bitcount of the result.
Definition: bmfunc.h:6540
bm::block_find_rank
SIZE_TYPE block_find_rank(const bm::word_t *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
Find rank in block (GAP or BIT)
Definition: bmfunc.h:7761
bm::set_SUB
@ set_SUB
Definition: bmconst.h:156
bm::set_block_bits_u64
void set_block_bits_u64(bm::word_t *BMRESTRICT block, const bm::id64_t *BMRESTRICT idx, bm::id64_t start, bm::id64_t stop) BMNOEXCEPT
set bits in a bit-block using global index
Definition: bmfunc.h:8605
bm::bit_block_change32
unsigned bit_block_change32(const bm::word_t *block, unsigned size) BMNOEXCEPT
Definition: bmfunc.h:4533
bm::gap_2_dgap
T * gap_2_dgap(const T *BMRESTRICT gap_buf, T *BMRESTRICT dgap_buf, bool copy_head=true) BMNOEXCEPT
Convert GAP buffer into D-GAP buffer.
Definition: bmfunc.h:2188
bm::set_COUNT
@ set_COUNT
Definition: bmconst.h:159
bm::bv_statistics::bit_blocks
size_t bit_blocks
Number of bit blocks.
Definition: bmfunc.h:56
bm::bit_block_any_range
bm::id_t bit_block_any_range(const bm::word_t *block, bm::word_t left, bm::word_t right) BMNOEXCEPT
bm::bit_invert
void bit_invert(T *start) BMNOEXCEPT
Definition: bmfunc.h:5253
bm::set_sub_total_bits
const unsigned set_sub_total_bits
Definition: bmconst.h:99
bm::all_set
Structure carries pointer on bit block with all bits 1.
Definition: bmfunc.h:842
bm::set_representation
set_representation
set representation variants
Definition: bmconst.h:201
bm::for_each_nzblock_range
void for_each_nzblock_range(T ***root, N top_size, N nb_from, N nb_to, F &f) BMNOEXCEPT
Definition: bmfunc.h:1325
VECT_BITCOUNT
#define VECT_BITCOUNT(first, last)
Definition: bmsse4.h:1719
bm::gap_convert_to_arr
D gap_convert_to_arr(D *BMRESTRICT dest, const T *BMRESTRICT buf, unsigned dest_len, bool invert=false) BMNOEXCEPT
Convert gap block into array of ints corresponding to 1 bits.
Definition: bmfunc.h:4320
bm::is_bits_one
bool is_bits_one(const bm::wordop_t *start) BMNOEXCEPT
Returns "true" if all bits in the block are 1.
Definition: bmfunc.h:5277
VECT_SHIFT_R1_AND
#define VECT_SHIFT_R1_AND(b, co, m, digest)
Definition: bmsse4.h:1806
bm::gap_free_elements
unsigned gap_free_elements(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returns number of free elements in GAP block array. Difference between GAP block capacity on this lev...
Definition: bmfunc.h:4063
VECT_BIT_FIND_DIFF
#define VECT_BIT_FIND_DIFF(src1, src2, pos)
Definition: bmsse4.h:1829
VECT_SHIFT_R1
#define VECT_SHIFT_R1(b, acc, co)
Definition: bmsse4.h:1803
bm::for_each_nzblock_if
bool for_each_nzblock_if(T ***root, BI size1, F &f) BMNOEXCEPT
Definition: bmfunc.h:1613
bm::bit_count_table
Structure to aid in counting bits table contains count of bits in 0-255 diapason of numbers.
Definition: bmconst.h:287
bm::gap_operation_any_and
BMFORCEINLINE unsigned gap_operation_any_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP AND operation test.
Definition: bmfunc.h:5670
BMGAP_PTR
#define BMGAP_PTR(ptr)
Definition: bmdef.h:179
BM_INCWORD_BITCOUNT
#define BM_INCWORD_BITCOUNT(cnt, w)
Definition: bmdef.h:386
bm::bit_OR
Bit OR functor.
Definition: bmfunc.h:8222
bm::ptrp_test
bm::id64_t ptrp_test(ptr_payload_t ptr, bm::gap_word_t v) BMNOEXCEPT
Definition: bmfunc.h:8857
bm::bit_count_nonzero_size
unsigned bit_count_nonzero_size(const T *blk, unsigned data_size) BMNOEXCEPT
Inspects block for full zero words.
Definition: bmfunc.h:7407
bm::block_is_interval
bool block_is_interval(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if all bits are 1 in the block [left, right] and border bits are 0.
Definition: bmfunc.h:5324
bm::gap_count_and
unsigned gap_count_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount AND operation test.
Definition: bmfunc.h:5687
VECT_XOR_BLOCK_2WAY
#define VECT_XOR_BLOCK_2WAY(dst, src1, src2)
Definition: bmsse4.h:1773
bm::bit_COUNT_AND::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8259
VECT_OR_BLOCK
#define VECT_OR_BLOCK(dst, src)
Definition: bmsse4.h:1749
bm::gap_capacity
unsigned gap_capacity(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returs GAP block capacity.
Definition: bmfunc.h:1114
bm::get_block_coord
void get_block_coord(BI_TYPE nb, unsigned &i, unsigned &j) BMNOEXCEPT
Recalc linear bvector block index into 2D matrix coordinates.
Definition: bmfunc.h:147
bm::gap_operation_to_bitset_func_type
void(* gap_operation_to_bitset_func_type)(unsigned *, const gap_word_t *)
Definition: bmfunc.h:8335
bm::bit_block_t
bit-block array wrapped into union for correct interpretation of 32-bit vs 64-bit access vs SIMD
Definition: bmutil.h:56
bm::set_block_digest_pos_shift
const unsigned set_block_digest_pos_shift
Definition: bmconst.h:67
bm::widx_to_digest_mask
BMFORCEINLINE bm::id64_t widx_to_digest_mask(unsigned w_idx) BMNOEXCEPT
Compute digest mask for word address in block.
Definition: bmfunc.h:650
bm::id64_t
unsigned long long int id64_t
Definition: bmconst.h:34
bm::decoder_range_adapter::decoder_range_adapter
decoder_range_adapter(DEC &dec, unsigned from_idx, unsigned to_idx)
Definition: bmfunc.h:8171
bm::bit_block_shift_l1_unr
bool bit_block_shift_l1_unr(bm::word_t *block, bm::word_t *empty_acc, bm::word_t co_flag) BMNOEXCEPT
Left bit-shift of bit-block by 1 bit (loop unrolled)
Definition: bmfunc.h:5010
bm::gap_calc_level
int gap_calc_level(unsigned len, const T *glevel_len) BMNOEXCEPT
Calculates GAP block capacity level.
Definition: bmfunc.h:4042
bm::gap_invert
void gap_invert(T *buf) BMNOEXCEPT
Inverts all bits in the GAP buffer.
Definition: bmfunc.h:4001
bm::bit_operation_and_any
bm::id_t bit_operation_and_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock AND operation test.
Definition: bmfunc.h:6564
bm::gap_sub_to_bitset
void gap_sub_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
SUB (AND NOT) GAP block to bitblock.
Definition: bmfunc.h:3320
bm::gap_find_rank
SIZE_TYPE gap_find_rank(const T *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
GAP block find position for the rank.
Definition: bmfunc.h:2049
bm::word_bitcount64
BMFORCEINLINE unsigned word_bitcount64(bm::id64_t x) BMNOEXCEPT
Definition: bmfunc.h:230
bm::all_set::_block
static all_set_block _block
Definition: bmfunc.h:901
VECT_ANDNOT_ARR_2_MASK
#define VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, mask)
Definition: bmsse4.h:1716
bm::gap_bitset_xor_count
bm::id_t gap_bitset_xor_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block XOR masked by GAP block.
Definition: bmfunc.h:3714
bm::gap_convert_to_bitset_smart
unsigned * gap_convert_to_bitset_smart(unsigned *BMRESTRICT dest, const T *BMRESTRICT buf, id_t set_max) BMNOEXCEPT
Smart GAP block to bitblock conversion.
Definition: bmfunc.h:3881
bm::set_word_shift
const unsigned set_word_shift
Definition: bmconst.h:71
BMNOEXCEPT2
#define BMNOEXCEPT2
Definition: bmdef.h:99
bm::sub_op
BMFORCEINLINE unsigned sub_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP or functor.
Definition: bmfunc.h:5623
bm::gap_and_to_bitset
void gap_and_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
ANDs GAP block to bitblock.
Definition: bmfunc.h:3486
bm::set_sub_array_size
const unsigned set_sub_array_size
Definition: bmconst.h:94
bm::gap_find_first
unsigned gap_find_first(const T *BMRESTRICT buf, unsigned *BMRESTRICT first) BMNOEXCEPT
GAP block find the first set bit.
Definition: bmfunc.h:1193
bm::wordop_t
id64_t wordop_t
Definition: bmconst.h:127
bm::bit_ASSIGN::operator()
W operator()(W, W w2) BMNOEXCEPT
Definition: bmfunc.h:8242
bm::digest_mask
BMFORCEINLINE bm::id64_t digest_mask(unsigned from, unsigned to) BMNOEXCEPT
Compute digest mask for [from..to] positions.
Definition: bmfunc.h:665
bm::check_block_one
bool check_block_one(const bm::word_t *blk, bool deep_scan) BMNOEXCEPT
Checks if block has only 1 bits.
Definition: bmfunc.h:7882
bm::bit_block_insert
bm::word_t bit_block_insert(bm::word_t *BMRESTRICT block, unsigned bitpos, bool value) BMNOEXCEPT
insert bit into position and shift the rest right with carryover
Definition: bmfunc.h:4876
bm::xor_op
BMFORCEINLINE unsigned xor_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP xor functor.
Definition: bmfunc.h:5610
bm::bit_XOR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8236
bm::bv_statistics::gap_blocks
size_t gap_blocks
Number of GAP blocks.
Definition: bmfunc.h:57
bm::gap_overhead
unsigned gap_overhead(const T *length, const T *length_end, const T *glevel_len) BMNOEXCEPT
Calculates memory overhead for number of gap blocks sharing the same memory allocation table (level l...
Definition: bmfunc.h:7905
bm::bitcmp
int bitcmp(const T *buf1, const T *buf2, unsigned len) BMNOEXCEPT
Lexicographical comparison of BIT buffers.
Definition: bmfunc.h:4081
bm::bitblock_get_adapter::bitblock_get_adapter
bitblock_get_adapter(const bm::word_t *bit_block)
Definition: bmfunc.h:8124
bm::all_set::all_set_block::all_set_block
all_set_block()
Definition: bmfunc.h:850
bm::all_set::is_full_block
static BMFORCEINLINE bool is_full_block(const bm::word_t *bp) BMNOEXCEPT
Definition: bmfunc.h:894
bm::sub_bit_block
void sub_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
SUB (AND NOT) bit interval to 1 in the bitblock.
Definition: bmfunc.h:3217
bm::bit_decode_cache::bits
unsigned short bits[65]
Definition: bmfunc.h:134
bm::bitscan
unsigned short bitscan(V w, B *bits) BMNOEXCEPT
Definition: bmfunc.h:554
bm::bv_statistics::gap_cap_overhead
size_t gap_cap_overhead
gap memory overhead between length and capacity
Definition: bmfunc.h:62
bm::set_bitscan_wave_size
const unsigned short set_bitscan_wave_size
Size of bit decode wave in words.
Definition: bmfunc.h:8421
bm::check_block_zero
bool check_block_zero(const bm::word_t *blk, bool deep_scan) BMNOEXCEPT
Checks all conditions and returns true if block consists of only 0 bits.
Definition: bmfunc.h:7858
bm::bit_ASSIGN
Bit ASSIGN functor.
Definition: bmfunc.h:8240
bm::bit_XOR
Bit XOR functor.
Definition: bmfunc.h:8234
bm::sse2_gap_sum_arr
const bm::gap_word_t * sse2_gap_sum_arr(const bm::gap_word_t *BMRESTRICT pbuf, unsigned sse_vect_waves, unsigned *sum)
Gap block population count (array sum) utility.
Definition: bmsse_util.h:891
bm::sum_arr
bm::id64_t sum_arr(const T *first, const T *last) BMNOEXCEPT
Definition: bmfunc.h:1695
bm::word_select64_bitscan
unsigned word_select64_bitscan(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition: bmfunc.h:597
bm::word_select64_linear
unsigned word_select64_linear(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition: bmfunc.h:574
bm::xor_bit_block
void xor_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
XOR bit interval to 1 in the bitblock.
Definition: bmfunc.h:3264
bm::parallel_popcnt_32
int parallel_popcnt_32(unsigned int n) BMNOEXCEPT
Definition: bmfunc.h:215
bm::bit_block_and
bm::id64_t bit_block_and(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock AND operation. Function does not analyse availability of source and destination blocks...
Definition: bmfunc.h:5940
bm::operation_functions::gap2bit_table_
static gap_operation_to_bitset_func_type gap2bit_table_[bm::set_END]
Definition: bmfunc.h:8353
bm::gap_is_interval
bool gap_is_interval(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if any bits are 1 in GAP buffer in the [left, right] range and flanked with 0s.
Definition: bmfunc.h:1962
bm::bit_block_or
bool bit_block_or(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock OR operation. Function does not analyse availability of source and destination blocks.
Definition: bmfunc.h:6749
bm::all_bits_mask
const id64_t all_bits_mask
Definition: bmconst.h:128
bm::bit_block_shift_r1_and
bool bit_block_shift_r1_and(bm::word_t *BMRESTRICT block, bm::word_t co_flag, const bm::word_t *BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest) BMNOEXCEPT
Right bit-shift of bit-block by 1 bit (reference) + AND.
Definition: bmfunc.h:5089
bm::bit_COUNT_XOR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8270
bm::bit_COUNT_SUB_AB::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8293
bm::sse4_bit_block_gather_scatter
void sse4_bit_block_gather_scatter(unsigned *BMRESTRICT arr, const unsigned *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx)
Definition: bmsse4.h:1343
bm::bit_list_4
unsigned bit_list_4(T w, B *bits) BMNOEXCEPT
Unpacks word into list of ON bit indexes (quad-bit based)
Definition: bmfunc.h:456
bm::bitblock_sum_adapter::sum
bm::word_t sum() const BMNOEXCEPT
Get accumulated sum.
Definition: bmfunc.h:8158
bm::gap_add_value
unsigned gap_add_value(T *buf, unsigned pos) BMNOEXCEPT
Add new value to the end of GAP buffer.
Definition: bmfunc.h:2827
bm::bv_statistics::add_bit_block
void add_bit_block() BMNOEXCEPT
cound bit block
Definition: bmfunc.h:67
bm::pair
Pair type.
Definition: bmfunc.h:121
bm::bit_COUNT_SUB_BA
Bit SUB BA functor.
Definition: bmfunc.h:8302
bm::for_each_gap_dbit
void for_each_gap_dbit(const T *buf, F &func)
Iterate gap block as delta-bits with a functor.
Definition: bmfunc.h:4265
bm::set_OR
@ set_OR
Definition: bmconst.h:155
bm::idx_arr_block_lookup_u64
bm::id64_t idx_arr_block_lookup_u64(const bm::id64_t *idx, bm::id64_t size, bm::id64_t nb, bm::id64_t start) BMNOEXCEPT
block boundaries look ahead U32
Definition: bmfunc.h:8546
bm::set_block_mask
const unsigned set_block_mask
Definition: bmconst.h:56
bm::bit_block_or_5way
bool bit_block_or_5way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, const bm::word_t *BMRESTRICT src3, const bm::word_t *BMRESTRICT src4) BMNOEXCEPT
5 way (target, source1, source2) bitblock OR operation. Function does not analyse availability of sou...
Definition: bmfunc.h:6910
bm::bit_COUNT_OR
Bit COUNT OR functor.
Definition: bmfunc.h:8279
bm::set_array_mask
const unsigned set_array_mask
Definition: bmconst.h:96
bm::gap_bit_count_unr
unsigned gap_bit_count_unr(const T *buf) BMNOEXCEPT
Calculates number of bits ON in GAP buffer. Loop unrolled version.
Definition: bmfunc.h:1801
VECT_AND_DIGEST_5WAY
#define VECT_AND_DIGEST_5WAY(dst, src1, src2, src3, src4)
Definition: bmsse4.h:1743
bm::block_ptr_array_range
bool block_ptr_array_range(bm::word_t **arr, unsigned &left, unsigned &right) BMNOEXCEPT
array range detector
Definition: bmfunc.h:8662
VECT_GAP_BFIND
#define VECT_GAP_BFIND(buf, pos, is_set)
Definition: bmsse4.h:1835
bm::operation_functions::bit_op_count_table_
static bit_operation_count_func_type bit_op_count_table_[bm::set_END]
Definition: bmfunc.h:8357
bm::bitblock_store_adapter::push_back
BMFORCEINLINE void push_back(bm::word_t w)
Definition: bmfunc.h:8142
bm::gap_split
void gap_split(const T *buf, T *arr0, T *arr1, T &arr0_cnt, T &arr1_cnt) BMNOEXCEPT
Definition: bmfunc.h:1713
bm::xor_swap
void xor_swap(W &x, W &y) BMNOEXCEPT
XOR swap two scalar variables.
Definition: bmfunc.h:909
bm::gap_find_last
unsigned gap_find_last(const T *BMRESTRICT buf, unsigned *BMRESTRICT last) BMNOEXCEPT
GAP block find the last set bit.
Definition: bmfunc.h:1162
bm::bit_block_to_gap
unsigned bit_block_to_gap(gap_word_t *BMRESTRICT dest, const unsigned *BMRESTRICT block, unsigned dest_len) BMNOEXCEPT
Converts bit block to GAP.
Definition: bmfunc.h:4163
bm::for_each_dgap
void for_each_dgap(const T *gap_buf, Func &func)
Definition: bmfunc.h:2147
bm::setop2op
bm::operation setop2op(bm::set_operation op) BMNOEXCEPT
Convert set operation to operation.
Definition: bmfunc.h:826
bm::gap_operation_dry_xor
bool gap_operation_dry_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, unsigned &dsize, unsigned limit) BMNOEXCEPT
Definition: bmfunc.h:5725
bm::gap_operation_any_sub
BMFORCEINLINE unsigned gap_operation_any_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP SUB operation test.
Definition: bmfunc.h:5860
bm::gap_set_all
void gap_set_all(T *buf, unsigned set_max, unsigned value) BMNOEXCEPT
Sets all bits to 0 or 1 (GAP)
Definition: bmfunc.h:3933
bm::gap_test
unsigned gap_test(const T *BMRESTRICT buf, unsigned pos) BMNOEXCEPT
Tests if bit = pos is true.
Definition: bmfunc.h:1255
bm::idx_arr_block_lookup_u32
unsigned idx_arr_block_lookup_u32(const unsigned *idx, unsigned size, unsigned nb, unsigned start) BMNOEXCEPT
block boundaries look ahead U32
Definition: bmfunc.h:8572
bm::bit_decode_cache::cvalue
bm::id64_t cvalue
Definition: bmfunc.h:136
bm::gap_block_find
unsigned gap_block_find(const T *BMRESTRICT buf, unsigned nbit, bm::id_t *BMRESTRICT prev) BMNOEXCEPT
Searches for the next 1 bit in the GAP block.
Definition: bmfunc.h:3098
bm::bit_convert_to_arr
T bit_convert_to_arr(T *BMRESTRICT dest, const unsigned *BMRESTRICT src, bm::id_t bits, unsigned dest_len, unsigned mask=0) BMNOEXCEPT
Convert bit block into an array of ints corresponding to 1 bits.
Definition: bmfunc.h:7822
bm::gap_buff_count_op
unsigned gap_buff_count_op(const T *vect1, const T *vect2, F f) BMNOEXCEPT2
Abstract distance(similarity) operation for GAP buffers. Receives functor F as a template argument.
Definition: bmfunc.h:2576
bm::gap_bit_count_to
unsigned gap_bit_count_to(const T *const buf, T right, bool is_corrected=false) BMNOEXCEPT
Counts 1 bits in GAP buffer in the closed [0, right] range.
Definition: bmfunc.h:2101
bm::gap_buff_any_op
unsigned gap_buff_any_op(const T *BMRESTRICT vect1, unsigned vect1_mask, const T *BMRESTRICT vect2, unsigned vect2_mask, F f) BMNOEXCEPT2
Abstract distance test operation for GAP buffers. Receives functor F as a template argument.
Definition: bmfunc.h:2507
bm::gap_buff_dry_op
bool gap_buff_dry_op(const T *BMRESTRICT vect1, const T *BMRESTRICT vect2, F &f, unsigned &dlen, unsigned limit) BMNOEXCEPT2
Abstract operation for GAP buffers (predicts legth) Receives functor F as a template argument.
Definition: bmfunc.h:2434
BMNOEXCEPT
#define BMNOEXCEPT
Definition: bmdef.h:79
VECT_SUB_DIGEST_2WAY
#define VECT_SUB_DIGEST_2WAY(dst, src1, src2)
Definition: bmsse4.h:1767
bm::bit_COUNT_B::operator()
W operator()(W, W w2) BMNOEXCEPT
Definition: bmfunc.h:8326
bm::copy_to_array_functor::ptr
B * ptr()
Definition: bmfunc.h:392
bm::bit_operation_or_any
bm::id_t bit_operation_or_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock OR operation test.
Definition: bmfunc.h:6716
bm::wordcmp
int wordcmp(T a, T b)
Lexicographical comparison of two words as bit strings. Auxiliary implementation for testing and refe...
Definition: bmfunc.h:1027
bm::bit_block_gather_scatter
void bit_block_gather_scatter(unsigned *BMRESTRICT arr, const bm::word_t *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx) BMNOEXCEPT
bit index to word gather-scatter algorithm (SIMD)
Definition: bmfunc.h:8471
bm::bit_block_or_3way
bool bit_block_or_3way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
3 way (target | source1 | source2) bitblock OR operation. Function does not analyse availability of s...
Definition: bmfunc.h:6866
bm::gap_bit_count_range
unsigned gap_bit_count_range(const T *const buf, unsigned left, unsigned right) BMNOEXCEPT
Counts 1 bits in GAP buffer in the closed [left, right] range.
Definition: bmfunc.h:1867
bm::decoder_range_adapter
Adapter to get words from a range stream (see range serialized bit-block)
Definition: bmfunc.h:8168
VECT_OR_BLOCK_5WAY
#define VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4)
Definition: bmsse4.h:1758
bm::set_block_digest_wave_size
const unsigned set_block_digest_wave_size
Definition: bmconst.h:66
bm::for_each_nzblock2
void for_each_nzblock2(T ***root, unsigned size1, F &f)
Definition: bmfunc.h:1491
bm::bitscan_popcnt64
unsigned short bitscan_popcnt64(bm::id64_t w, B *bits) BMNOEXCEPT
Unpacks 64-bit word into list of ON bit indexes using popcnt method.
Definition: bmfunc.h:518
bm::decoder_range_adapter::get_32
bm::word_t get_32() BMNOEXCEPT
Definition: bmfunc.h:8178
bm::gap_level
T gap_level(const T *BMRESTRICT buf) BMNOEXCEPT
Returs GAP blocks capacity level.
Definition: bmfunc.h:1144
VECT_AND_BLOCK
#define VECT_AND_BLOCK(dst, src)
Definition: bmsse4.h:1737
bm::all_set::all_set_block::_p_fullp
bm::word_t * _p_fullp
Definition: bmfunc.h:848
bm::bitblock_get_adapter::get_32
BMFORCEINLINE bm::word_t get_32() BMNOEXCEPT
Definition: bmfunc.h:8127
bm::update_block_digest0
bm::id64_t update_block_digest0(const bm::word_t *const block, bm::id64_t digest) BMNOEXCEPT
Compute digest for 64 non-zero areas based on existing digest (function revalidates zero areas)
Definition: bmfunc.h:776
bm::pair::pair
pair(First f, Second s)
Definition: bmfunc.h:126
bm::gap_set_value
unsigned gap_set_value(unsigned val, T *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set) BMNOEXCEPT
Sets or clears bit in the GAP buffer.
Definition: bmfunc.h:2661
bm::calc_block_digest0
bm::id64_t calc_block_digest0(const bm::word_t *const block) BMNOEXCEPT
Compute digest for 64 non-zero areas.
Definition: bmfunc.h:734
FULL_BLOCK_FAKE_ADDR
#define FULL_BLOCK_FAKE_ADDR
Definition: bmdef.h:149
bm::bit_OR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8224
bm::lower_bound_linear_u64
unsigned lower_bound_linear_u64(const unsigned long long *arr, unsigned long long target, unsigned from, unsigned to) BMNOEXCEPT
Linear lower bound search in unsigned LONG array.
Definition: bmfunc.h:8720
VECT_STREAM_BLOCK
#define VECT_STREAM_BLOCK(dst, src)
Definition: bmsse4.h:1779
bm::bit_operation_xor
bm::word_t * bit_operation_xor(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock XOR operation.
Definition: bmfunc.h:7311
bm::bit_COUNT_B
Bit COUNT B functor.
Definition: bmfunc.h:8324
VECT_BITCOUNT_XOR
#define VECT_BITCOUNT_XOR(first, last, mask)
Definition: bmsse4.h:1728
bm::gap_word_t
unsigned short gap_word_t
Definition: bmconst.h:77
bm::ptr_payload_t::blk
bm::word_t * blk
Definition: bmfunc.h:8847
bm::d_copy_func::operator()
void operator()(T dgap)
Definition: bmfunc.h:2169
bm::set_operation
set_operation
Codes of set operations.
Definition: bmconst.h:152
bm::bit_operation_or
bm::word_t * bit_operation_or(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Block OR operation. Makes analysis if block is 0 or FULL.
Definition: bmfunc.h:6959
bm::all_set::all_set_block::BM_VECT_ALIGN_ATTR
bm::word_t BM_VECT_ALIGN _p[bm::set_block_size] BM_VECT_ALIGN_ATTR
Definition: bmfunc.h:847
bm::test_bit
BMFORCEINLINE unsigned test_bit(const unsigned *block, unsigned bitpos) BMNOEXCEPT
Test 1 bit in a block.
Definition: bmfunc.h:3152
bm::all_set::all_set_block::BM_VECT_ALIGN_ATTR
bm::word_t BM_VECT_ALIGN *_s[bm::set_sub_array_size] BM_VECT_ALIGN_ATTR
Definition: bmfunc.h:846
bm::set_array_shift
const unsigned set_array_shift
Definition: bmconst.h:95
bm::block_any
bool block_any(const bm::word_t *const BMRESTRICT block) BMNOEXCEPT
Returns "true" if one bit is set in the block Function check for block varieties.
Definition: bmfunc.h:5584
bm::bit_decode_cache::bcnt
unsigned bcnt
Definition: bmfunc.h:135
bm::conditional
ad-hoc conditional expressions
Definition: bmutil.h:110
bm::gap_convert_to_bitset
void gap_convert_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT buf) BMNOEXCEPT
GAP block to bitblock conversion.
Definition: bmfunc.h:3859
bm::bit_recomb
void bit_recomb(It1 &it1, It2 &it2, BinaryOp &op, Encoder &enc, unsigned block_size=bm::set_block_size) BMNOEXCEPT
Definition: bmfunc.h:8201
bm::bit_SUB::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8230
bm::bit_list
unsigned bit_list(T w, B *bits) BMNOEXCEPT
Unpacks word into list of ON bit indexes.
Definition: bmfunc.h:438
BM_ASSERT
#define BM_ASSERT
Definition: bmdef.h:130
bm::block_set_table
Structure keeps all-left/right ON bits masks.
Definition: bmconst.h:342
bm::get_super_block_start
BMFORCEINLINE RTYPE get_super_block_start(unsigned i) BMNOEXCEPT
Compute bit address of the first bit in a superblock.
Definition: bmfunc.h:158
bm::ptr_payload_t::i64
bm::id64_t i64
Definition: bmfunc.h:8848
bm::gap_bitset_sub_count
bm::id_t gap_bitset_sub_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block SUB masked by GAP block.
Definition: bmfunc.h:3641
bm::operation_functions::gapop_table_
static gap_operation_func_type gapop_table_[bm::set_END]
Definition: bmfunc.h:8355
bm::bit_for_each_4
void bit_for_each_4(T w, F &func)
Templated algorithm to unpacks octet based word into list of ON bit indexes.
Definition: bmfunc.h:295
bm::bit_block_rotate_left_1_unr
void bit_block_rotate_left_1_unr(bm::word_t *block) BMNOEXCEPT
Unrolled cyclic rotation of bit-block left by 1 bit.
Definition: bmfunc.h:4840
bm::bv_statistics::add_gap_block
void add_gap_block(unsigned capacity, unsigned length) BMNOEXCEPT
count gap block
Definition: bmfunc.h:76
bm::id_t
unsigned int id_t
Definition: bmconst.h:37
VECT_LOWER_BOUND_SCAN_U32
#define VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to)
Definition: bmsse4.h:1797
bm::bitblock_sum_adapter
Bit-block sum adapter, takes values and sums it /internal.
Definition: bmfunc.h:8151
bm::bit_block_shift_r1
bool bit_block_shift_r1(bm::word_t *BMRESTRICT block, bm::word_t *BMRESTRICT empty_acc, bm::word_t co_flag) BMNOEXCEPT
Right bit-shift bitblock by 1 bit (reference)
Definition: bmfunc.h:4925
bm::bit_for_each
void bit_for_each(T w, F &func)
Templated algorithm to unpacks word into list of ON bit indexes.
Definition: bmfunc.h:365
VECT_INVERT_BLOCK
#define VECT_INVERT_BLOCK(first)
Definition: bmsse4.h:1734
bm::gap_is_all_zero
BMFORCEINLINE bool gap_is_all_zero(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Checks if GAP block is all-zero.
Definition: bmfunc.h:1072
bm::gap_operation_any_xor
BMFORCEINLINE unsigned gap_operation_any_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP XOR operation test.
Definition: bmfunc.h:5749
bm::gap_find_interval_end
bool gap_find_interval_end(const T *const BMRESTRICT buf, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the last 1 bit in the 111 interval of a GAP block.
Definition: bmfunc.h:1991
bm::set_XOR
@ set_XOR
Definition: bmconst.h:157
bm::all_set::all_set_block
Definition: bmfunc.h:844
VECT_OR_BLOCK_3WAY
#define VECT_OR_BLOCK_3WAY(dst, src1, src2)
Definition: bmsse4.h:1755
bm::bit_block_rotate_left_1
void bit_block_rotate_left_1(bm::word_t *block) BMNOEXCEPT
Definition: bmfunc.h:4824
bm::bitblock_store_adapter::bitblock_store_adapter
bitblock_store_adapter(bm::word_t *bit_block)
Definition: bmfunc.h:8140
bm::or_bit_block
void or_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
Sets bits to 1 in the bitblock.
Definition: bmfunc.h:3170
VECT_BITCOUNT_AND
#define VECT_BITCOUNT_AND(first, last, mask)
Definition: bmsse4.h:1722
bm::operation
operation
Bit operations.
Definition: bmconst.h:175
bmdef.h
Definitions(internal)
bm::bit_operation_sub_count
bm::id_t bit_operation_sub_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock SUB operation and calculates bitcount of the result.
Definition: bmfunc.h:6589
bm::bv_statistics::ptr_sub_blocks
size_t ptr_sub_blocks
Number of sub-blocks.
Definition: bmfunc.h:58
VECT_AND_DIGEST
#define VECT_AND_DIGEST(dst, src)
Definition: bmsse4.h:1740
bm::gap_operation_and
BMFORCEINLINE gap_word_t * gap_operation_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP AND operation.
Definition: bmfunc.h:5646
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx0, unsigned bit_idx1, unsigned bit_idx2, unsigned bit_idx3) BMNOEXCEPT
Definition: bmfunc.h:411
bm::set_bitset
@ set_bitset
Simple bitset.
Definition: bmconst.h:203
bm::is_const_set_operation
bool is_const_set_operation(set_operation op) BMNOEXCEPT
Returns true if set operation is constant (bitcount)
Definition: bmfunc.h:817
bm::bit_COUNT_A::operator()
W operator()(W w1, W) BMNOEXCEPT
Definition: bmfunc.h:8315
bm::bit_block_shift_r1_unr
bool bit_block_shift_r1_unr(bm::word_t *BMRESTRICT block, bm::word_t *BMRESTRICT empty_acc, bm::word_t co_flag) BMNOEXCEPT
Right bit-shift of bit-block by 1 bit (loop unrolled)
Definition: bmfunc.h:4955
VECT_SHIFT_L1
#define VECT_SHIFT_L1(b, acc, co)
Definition: bmsse4.h:1800
bm::bit_operation_sub
bm::word_t * bit_operation_sub(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock SUB operation.
Definition: bmfunc.h:7182
bm::bit_find_last
unsigned bit_find_last(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT last) BMNOEXCEPT
BIT block find the last set bit (backward search)
Definition: bmfunc.h:7518
bm::gap_test_unr
unsigned gap_test_unr(const T *BMRESTRICT buf, const unsigned pos) BMNOEXCEPT
Tests if bit = pos is true. Analog of bm::gap_test with SIMD unrolling.
Definition: bmfunc.h:1298
VECT_XOR_BLOCK
#define VECT_XOR_BLOCK(dst, src)
Definition: bmsse4.h:1770
bm::block_find_interval_end
unsigned block_find_interval_end(const bm::word_t *BMRESTRICT block, unsigned nbit_from, unsigned *BMRESTRICT found_nbit) BMNOEXCEPT
Find end of the current 111 interval.
Definition: bmfunc.h:5428
bm::bit_to_gap
unsigned bit_to_gap(gap_word_t *BMRESTRICT dest, const unsigned *BMRESTRICT block, unsigned dest_len) BMNOEXCEPT
Convert bit block to GAP representation.
Definition: bmfunc.h:4248
VECT_IS_DIGEST_ZERO
#define VECT_IS_DIGEST_ZERO(start)
Definition: bmsse4.h:1791
bm::block_any_range
bool block_any_range(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if one bit is set in the block [left, right] Function check for block varieties.
Definition: bmfunc.h:5563
VECT_BLOCK_CHANGE
#define VECT_BLOCK_CHANGE(block, size)
Definition: bmsse4.h:1815
bm::or_op
BMFORCEINLINE unsigned or_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP or functor.
Definition: bmfunc.h:5617
bm::gap_max_bits
const unsigned gap_max_bits
Definition: bmconst.h:80
bm::improve_gap_levels
bool improve_gap_levels(const T *length, const T *length_end, T *glevel_len) BMNOEXCEPT
Finds optimal gap blocks lengths.
Definition: bmfunc.h:7932
bm::word_bitcount
BMFORCEINLINE bm::id_t word_bitcount(bm::id_t w) BMNOEXCEPT
Definition: bmfunc.h:197
bm::gap_add_to_bitset
void gap_add_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr, unsigned len) BMNOEXCEPT
Adds(OR) GAP block to bitblock.
Definition: bmfunc.h:3436
bm::lower_bound_u64
unsigned lower_bound_u64(const unsigned long long *arr, unsigned long long target, unsigned from, unsigned to) BMNOEXCEPT
Hybrid, binary-linear lower bound search in unsigned LONG array.
Definition: bmfunc.h:8780
bm::bit_block_xor_any
unsigned bit_block_xor_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function XORs two bitblocks and and tests for any bit. Function does not analyse availability of sour...
Definition: bmfunc.h:6292
IS_VALID_ADDR
#define IS_VALID_ADDR(addr)
Definition: bmdef.h:152
bm::bit_find_first_if_1
bool bit_find_first_if_1(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT first, bm::id64_t digest) BMNOEXCEPT
BIT block find the first set bit if only 1 bit is set.
Definition: bmfunc.h:7625
bm::lower_bound_u32
unsigned lower_bound_u32(const unsigned *arr, unsigned target, unsigned from, unsigned to) BMNOEXCEPT
Hybrid, binary-linear lower bound search in unsigned array.
Definition: bmfunc.h:8745
bm::gap_bitset_and_count
bm::id_t gap_bitset_and_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT pcurr) BMNOEXCEPT
Compute bitcount of bit block AND masked by GAP block.
Definition: bmfunc.h:3583
bm::block_is_all_one_range
bool block_is_all_one_range(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if all bits are 1 in the block [left, right] Function check for block varieties.
Definition: bmfunc.h:5303
VECT_SET_BLOCK
#define VECT_SET_BLOCK(dst, value)
Definition: bmsse4.h:1782
bm::bit_count_change
bm::id_t bit_count_change(bm::word_t w) BMNOEXCEPT
Definition: bmfunc.h:4517
bm::bit_SUB
Bit SUB functor.
Definition: bmfunc.h:8228
FULL_BLOCK_REAL_ADDR
#define FULL_BLOCK_REAL_ADDR
Definition: bmdef.h:148
bmutil.h
Bit manipulation primitives (internal)
VECT_BLOCK_SET_DIGEST
#define VECT_BLOCK_SET_DIGEST(dst, val)
Definition: bmsse4.h:1794
bm::bv_statistics::reset
void reset() BMNOEXCEPT
Reset statisctics.
Definition: bmfunc.h:96
bm::gap_set_array
unsigned gap_set_array(T *buf, const T *arr, unsigned len) BMNOEXCEPT
Convert array to GAP buffer.
Definition: bmfunc.h:3005
VECT_SET_BLOCK_BITS
#define VECT_SET_BLOCK_BITS(block, idx, start, stop)
Definition: bmsse4.h:1812
bm::bit_COUNT_SUB_BA::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8304
bm::bit_block_and_count
unsigned bit_block_and_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function ANDs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition: bmfunc.h:6164
VECT_BIT_FIND_FIRST
#define VECT_BIT_FIND_FIRST(src, pos)
Definition: bmsse4.h:1826
bm::gap_init_range_block
void gap_init_range_block(T *buf, T from, T to, T value) BMNOEXCEPT
Init gap block so it has block in it (can be whole block)
Definition: bmfunc.h:3951
bm::bv_statistics::bv_count
size_t bv_count
Number of bit-vectors.
Definition: bmfunc.h:59
bm::bit_array_compute_gaps
unsigned bit_array_compute_gaps(const T *arr, unsigned len) BMNOEXCEPT
Compute number of GAPs in bit-array.
Definition: bmfunc.h:3066
bm::bit_COUNT_SUB_AB
Bit COUNT SUB AB functor.
Definition: bmfunc.h:8291
bm::gap_bfind
unsigned gap_bfind(const T *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set) BMNOEXCEPT
Definition: bmfunc.h:1220
bm::bit_COUNT_A
Bit COUNT A functor.
Definition: bmfunc.h:8313
bm::best_representation
bm::set_representation best_representation(unsigned bit_count, unsigned total_possible_bitcount, unsigned gap_count, unsigned block_size) BMNOEXCEPT
Choose best representation for a bit-block.
Definition: bmfunc.h:7785
bm::set_END
@ set_END
Definition: bmconst.h:168
bm::bit_block_find
unsigned bit_block_find(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the next 1 bit in the BIT block.
Definition: bmfunc.h:7467
bm::bit_operation_or_count
bm::id_t bit_operation_or_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock OR operation and calculates bitcount of the result.
Definition: bmfunc.h:6679
bm::pair::second
Second second
Definition: bmfunc.h:124
bm::bit_operation_and
bm::word_t * bit_operation_and(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock AND operation.
Definition: bmfunc.h:6476
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx0, unsigned bit_idx1, unsigned bit_idx2) BMNOEXCEPT
Definition: bmfunc.h:403
bm::dgap_2_gap
void dgap_2_gap(const T *BMRESTRICT dgap_buf, T *BMRESTRICT gap_buf, T gap_header=0) BMNOEXCEPT
Convert D-GAP buffer into GAP buffer.
Definition: bmfunc.h:2214
bm::bit_COUNT
Bit COUNT functor.
Definition: bmfunc.h:8246
bm::bit_scan_reverse32
unsigned bit_scan_reverse32(unsigned value) BMNOEXCEPT
Definition: bmutil.h:301
bm::set_block_shift
const unsigned set_block_shift
Definition: bmconst.h:55
bm::bit_operation_count_func_type
bm::id_t(* bit_operation_count_func_type)(const bm::word_t *BMRESTRICT, const bm::word_t *BMRESTRICT)
Definition: bmfunc.h:8345
bm::gap_limit
unsigned gap_limit(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returs GAP block capacity limit.
Definition: bmfunc.h:1130
bm::sse2_gap_test
unsigned sse2_gap_test(const unsigned short *BMRESTRICT buf, unsigned pos)
Hybrid binary search, starts as binary, then switches to scan.
Definition: bmsse2.h:461
bm::bit_block_or_any
unsigned bit_block_or_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function ORs two bitblocks and and tests for any bit. Function does not analyse availability of sourc...
Definition: bmfunc.h:6444
bm::d_copy_func::d_copy_func
d_copy_func(T *dg_buf)
Definition: bmfunc.h:2168
bm::bit_block_and_any
unsigned bit_block_and_any(const bm::word_t *src1, const bm::word_t *src2) BMNOEXCEPT
Function ANDs two bitblocks and tests for any bit. Function does not analyse availability of source b...
Definition: bmfunc.h:6213
bm::bit_block_find_interval_start
bool bit_block_find_interval_start(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the first 1 bit in the 111 interval of a BIT block.
Definition: bmfunc.h:5469
bm::bit_block_erase
void bit_block_erase(bm::word_t *block, unsigned bitpos, bool carry_over) BMNOEXCEPT
erase bit from position and shift the rest right with carryover
Definition: bmfunc.h:5033
bm::clear_bit
BMFORCEINLINE void clear_bit(unsigned *dest, unsigned bitpos) BMNOEXCEPT
Set 1 bit in a block.
Definition: bmfunc.h:3138
bm::count_leading_zeros_u64
unsigned count_leading_zeros_u64(bm::id64_t w) BMNOEXCEPT
64-bit bit-scan reverse
Definition: bmutil.h:353
bm::bit_COUNT_AND
Bit COUNT AND functor.
Definition: bmfunc.h:8257
bm::bitblock_get_adapter
Bit-block get adapter, takes bitblock and represents it as a get_32() accessor function.
Definition: bmfunc.h:8121
BMFORCEINLINE
#define BMFORCEINLINE
Definition: bmdef.h:203
bm
Definition: bm.h:76
bm::wordcmp0
int wordcmp0(T w1, T w2)
Lexicographical comparison of two words as bit strings (reference) Auxiliary implementation for testi...
Definition: bmfunc.h:998
VECT_COPY_BLOCK
#define VECT_COPY_BLOCK(dst, src)
Definition: bmsse4.h:1776
bm::gap_bitset_or_any
bm::id_t gap_bitset_or_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block OR masked by GAP block.
Definition: bmfunc.h:3822
IS_FULL_BLOCK
#define IS_FULL_BLOCK(addr)
Definition: bmdef.h:153
bm::bit_block_xor_count
unsigned bit_block_xor_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function XORs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition: bmfunc.h:6243
bm::word_select64
unsigned word_select64(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition: bmfunc.h:624
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx0, unsigned bit_idx1) BMNOEXCEPT
Definition: bmfunc.h:396
bm::bit_block_shift_r1_and_unr
bool bit_block_shift_r1_and_unr(bm::word_t *BMRESTRICT block, bm::word_t co_flag, const bm::word_t *BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest) BMNOEXCEPT
Right bit-shift bitblock by 1 bit (reference) + AND.
Definition: bmfunc.h:5161
bm::gap_operation_xor
gap_word_t * gap_operation_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP XOR operation.
Definition: bmfunc.h:5712
bm::bv_statistics::gap_levels
gap_word_t gap_levels[bm::gap_levels]
GAP block lengths in the bvect.
Definition: bmfunc.h:63
bm::bit_find_first_diff
bool bit_find_first_diff(const bm::word_t *BMRESTRICT blk1, const bm::word_t *BMRESTRICT blk2, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two bit-blocks.
Definition: bmfunc.h:4106
bm::operation_functions::gap_op_to_bit
static gap_operation_to_bitset_func_type gap_op_to_bit(unsigned i)
Definition: bmfunc.h:8360
VECT_AND_DIGEST_2WAY
#define VECT_AND_DIGEST_2WAY(dst, src1, src2)
Definition: bmsse4.h:1746
bm::bit_block_or_count
unsigned bit_block_or_count(const bm::word_t *src1, const bm::word_t *src2) BMNOEXCEPT
Function ORs two bitblocks and computes the bitcount. Function does not analyse availability of sourc...
Definition: bmfunc.h:6396
bm::bit_operation_xor_count
bm::id_t bit_operation_xor_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock XOR operation and calculates bitcount of the result.
Definition: bmfunc.h:7349
bm::bitblock_store_adapter
Bit-block store adapter, takes bitblock and saves results into it.
Definition: bmfunc.h:8137
bm::bv_statistics::memory_used
size_t memory_used
memory usage for all blocks and service tables
Definition: bmfunc.h:61
bm::gap_count_sub
BMFORCEINLINE unsigned gap_count_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount SUB (AND NOT) operation test.
Definition: bmfunc.h:5877
bm::bit_scan_reverse
unsigned bit_scan_reverse(T value) BMNOEXCEPT
Definition: bmutil.h:415
bm::block_find_first_diff
bool block_find_first_diff(const bm::word_t *BMRESTRICT blk, const bm::word_t *BMRESTRICT arg_blk, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two blocks (GAP or bit)
Definition: bmfunc.h:8030
VECT_IS_ZERO_BLOCK
#define VECT_IS_ZERO_BLOCK(dst)
Definition: bmsse4.h:1785
bm::bitscan_wave
unsigned short bitscan_wave(const bm::word_t *BMRESTRICT w_ptr, unsigned char *BMRESTRICT bits) BMNOEXCEPT
Unpacks word wave (Nx 32-bit words)
Definition: bmfunc.h:8433
VECT_BITCOUNT_SUB
#define VECT_BITCOUNT_SUB(first, last, mask)
Definition: bmsse4.h:1731
bm::operation_functions
Definition: bmfunc.h:8350
bm::set_array0
@ set_array0
array of 0 values
Definition: bmconst.h:206
bm::bit_AND::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8218
bm::bit_block_and_2way
bm::id64_t bit_block_and_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, bm::id64_t digest) BMNOEXCEPT
digest based bit-block AND
Definition: bmfunc.h:6101
bm::bit_operation_sub_any
bm::id_t bit_operation_sub_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock test of SUB operation.
Definition: bmfunc.h:6644
bm::bit_block_xor
bm::id64_t bit_block_xor(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock XOR operation. Function does not analyse availability of source and destination blocks...
Definition: bmfunc.h:7240
bm::copy_to_array_functor
Adaptor to copy 1 bits to array.
Definition: bmfunc.h:386
bm::check_zero_digest
bool check_zero_digest(bm::id64_t digest, unsigned bitpos_from, unsigned bitpos_to) BMNOEXCEPT
check if all digest bits for the range [from..to] are 0
Definition: bmfunc.h:688
bm::set_word_mask
const unsigned set_word_mask
Definition: bmconst.h:72
bm::bit_block_copy
void bit_block_copy(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Bitblock copy operation.
Definition: bmfunc.h:5898
bm::bit_block_calc_count_to
bm::id_t bit_block_calc_count_to(const bm::word_t *block, bm::word_t right) BMNOEXCEPT
Definition: bmfunc.h:4767
bm::word_t
unsigned int word_t
Definition: bmconst.h:38
bm::ptr_payload_t::i16
unsigned short i16[4]
Definition: bmfunc.h:8849
BMRESTRICT
#define BMRESTRICT
Definition: bmdef.h:193
bm::gap_length
BMFORCEINLINE bm::gap_word_t gap_length(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Returs GAP block length.
Definition: bmfunc.h:1098
bm::bit_decode_cache::bit_decode_cache
bit_decode_cache()
Definition: bmfunc.h:138
VECT_IS_ONE_BLOCK
#define VECT_IS_ONE_BLOCK(dst)
Definition: bmsse4.h:1788
bm::bit_block_stream
void bit_block_stream(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Bitblock copy/stream operation.
Definition: bmfunc.h:5917
bm::set_gap
@ set_gap
GAP-RLE compression.
Definition: bmconst.h:204
bm::bit_block_calc_count_range
bm::id_t bit_block_calc_count_range(const bm::word_t *block, bm::word_t left, bm::word_t right) BMNOEXCEPT
Definition: bmfunc.h:4698
bm::gap_is_all_one
BMFORCEINLINE bool gap_is_all_one(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Checks if GAP block is all-one.
Definition: bmfunc.h:1085
bm::bit_block_or_2way
bool bit_block_or_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
2 way (target := source1 | source2) bitblock OR operation.
Definition: bmfunc.h:6786
bm::bit_block_shift_l1
bool bit_block_shift_l1(bm::word_t *block, bm::word_t *empty_acc, bm::word_t co_flag) BMNOEXCEPT
Left bit-shift bitblock by 1 bit (reference)
Definition: bmfunc.h:4980
bm::bit_block_calc_change
unsigned bit_block_calc_change(const bm::word_t *block) BMNOEXCEPT
Definition: bmfunc.h:4607
bm::bit_block_sub_any
unsigned bit_block_sub_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function SUBs two bitblocks and and tests for any bit. Function does not analyse availability of sour...
Definition: bmfunc.h:6367
bm::gap_shift_r1
bool gap_shift_r1(T *BMRESTRICT buf, unsigned co_flag, unsigned *BMRESTRICT new_len) BMNOEXCEPT
Right shift GAP block by 1 bit.
Definition: bmfunc.h:2897
bm::for_each_block
void for_each_block(T ***root, unsigned size1, F &f, BLOCK_IDX start)
Definition: bmfunc.h:1647
bm::bit_block_and_5way
bm::id64_t bit_block_and_5way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src0, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, const bm::word_t *BMRESTRICT src3, bm::id64_t digest) BMNOEXCEPT
digest based bit-block AND 5-way
Definition: bmfunc.h:6034
bm::bmi_bslr_u64
BMFORCEINLINE unsigned long long bmi_bslr_u64(unsigned long long w) BMNOEXCEPT
Definition: bmutil.h:332
bm::d_copy_func::dgap_buf_
T * dgap_buf_
Definition: bmfunc.h:2171
bm::bitscan_popcnt
unsigned short bitscan_popcnt(bm::id_t w, B *bits, unsigned short offs) BMNOEXCEPT
Unpacks word into list of ON bit indexes using popcnt method.
Definition: bmfunc.h:475
bm::bit_find_rank
SIZE_TYPE bit_find_rank(const bm::word_t *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
BIT block find position for the rank.
Definition: bmfunc.h:7682
bm::for_each_nzblock
void for_each_nzblock(T ***root, unsigned size1, F &f)
Definition: bmfunc.h:1382
bm::bv_statistics
Structure with statistical information about memory allocation footprint, serialization projection,...
Definition: bmfunc.h:54
bm::bit_COUNT_OR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8281
bm::gap_bit_count
unsigned gap_bit_count(const T *buf, unsigned dsize=0) BMNOEXCEPT
Calculates number of bits ON in GAP buffer.
Definition: bmfunc.h:1773
VECT_SUB_DIGEST
#define VECT_SUB_DIGEST(dst, src)
Definition: bmsse4.h:1764
bm::bit_block_sub_2way
bm::id64_t bit_block_sub_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, bm::id64_t digest) BMNOEXCEPT
digest based bitblock SUB (AND NOT) operation (3 operand)
Definition: bmfunc.h:7115
bm::all_set::block_type
static bm::id64_t block_type(const bm::word_t *bp) BMNOEXCEPT
Definition: bmfunc.h:873
bm::bit_AND
Bit AND functor.
Definition: bmfunc.h:8216
bm::gap_bitset_and_any
bm::id_t gap_bitset_and_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT pcurr) BMNOEXCEPT
Bitcount test of bit block AND masked by GAP block.
Definition: bmfunc.h:3611
bm::bv_statistics::add
void add(const bv_statistics &st) BMNOEXCEPT
Sum data from another sttructure.
Definition: bmfunc.h:105
bm::gap_buff_op
void gap_buff_op(T *BMRESTRICT dest, const T *BMRESTRICT vect1, unsigned vect1_mask, const T *BMRESTRICT vect2, unsigned vect2_mask, F &f, unsigned &dlen) BMNOEXCEPT2
Abstract operation for GAP buffers. Receives functor F as a template argument.
Definition: bmfunc.h:2355
bm::set_array1
@ set_array1
array of set 1 values
Definition: bmconst.h:205
bm::get_block_start
BMFORCEINLINE RTYPE get_block_start(unsigned i, unsigned j) BMNOEXCEPT
Compute bit address of the first bit in a block.
Definition: bmfunc.h:168
bm::gap_bitset_sub_any
bm::id_t gap_bitset_sub_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block SUB masked by GAP block.
Definition: bmfunc.h:3676
bm::bmfor_each
F bmfor_each(T first, T last, F f)
Definition: bmfunc.h:1682
bm::set_bit
BMFORCEINLINE void set_bit(unsigned *dest, unsigned bitpos) BMNOEXCEPT
Set 1 bit in a block.
Definition: bmfunc.h:3125
bm::bit_block_is_all_one_range
bool bit_block_is_all_one_range(const bm::word_t *const BMRESTRICT block, bm::word_t left, bm::word_t right) BMNOEXCEPT
Definition: bmfunc.h:4621
bm::and_op
BMFORCEINLINE unsigned and_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP and functor.
Definition: bmfunc.h:5603
bm::sse42_test_all_zero_wave
BMFORCEINLINE bool sse42_test_all_zero_wave(const void *ptr)
check if wave of pointers is all NULL
Definition: bmsse4.h:595
bm::gap_find_first_diff
bool gap_find_first_diff(const T *BMRESTRICT buf1, const T *BMRESTRICT buf2, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two GAP-blocks.
Definition: bmfunc.h:2314
bm::gap_levels
const unsigned gap_levels
Definition: bmconst.h:84
bm::gap_operation_or
gap_word_t * gap_operation_or(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP OR operation.
Definition: bmfunc.h:5790
bm::bit_block_set
void bit_block_set(bm::word_t *BMRESTRICT dst, bm::word_t value) BMNOEXCEPT
Bitblock memset operation.
Definition: bmfunc.h:3841
bm::block_init_digest0
void block_init_digest0(bm::word_t *const block, bm::id64_t digest) BMNOEXCEPT
Init block with 000111000 pattren based on digest.
Definition: bmfunc.h:704
bm::gap_bitset_xor_any
bm::id_t gap_bitset_xor_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block XOR masked by GAP block.
Definition: bmfunc.h:3752
bm::bitblock_sum_adapter::push_back
BMFORCEINLINE void push_back(bm::word_t w) BMNOEXCEPT
Definition: bmfunc.h:8156
bm::all_set::is_valid_block_addr
static BMFORCEINLINE bool is_valid_block_addr(const bm::word_t *bp) BMNOEXCEPT
Definition: bmfunc.h:898
bm::bit_COUNT_XOR
Bit COUNT XOR functor.
Definition: bmfunc.h:8268
VECT_OR_BLOCK_2WAY
#define VECT_OR_BLOCK_2WAY(dst, src1, src2)
Definition: bmsse4.h:1752
VECT_ARR_BLOCK_LOOKUP
#define VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start)
Definition: bmsse4.h:1809
bm::gap_is_all_one_range
bool gap_is_all_one_range(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if all bits are 1 in GAP buffer in the [left, right] range.
Definition: bmfunc.h:1910
bm::bit_block_calc_count
bm::id_t bit_block_calc_count(const bm::word_t *block, const bm::word_t *block_end) BMNOEXCEPT
Bitcount for bit string.
Definition: bmfunc.h:4482
bm::gap_max_buff_len
const unsigned gap_max_buff_len
Definition: bmconst.h:79
bm::operation_functions::bit_operation_count
static bit_operation_count_func_type bit_operation_count(unsigned i)
Definition: bmfunc.h:8372
bm::bitcount64_4way
unsigned bitcount64_4way(bm::id64_t x, bm::id64_t y, bm::id64_t u, bm::id64_t v) BMNOEXCEPT
Definition: bmfunc.h:254