BitMagic-C++: bmsparsevec_serial.h Source File

BitMagic-C++

bmsparsevec_serial.h
Go to the documentation of this file.
1 #ifndef BMSPARSEVEC_SERIAL__H__INCLUDED__
2 #define BMSPARSEVEC_SERIAL__H__INCLUDED__
3 /*
4 Copyright(c) 2002-2017 Anatoliy Kuznetsov(anatoliy_kuznetsov at yahoo.com)
5
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
9
10 http://www.apache.org/licenses/LICENSE-2.0
11
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17
18 For more information please visit: http://bitmagic.io
19 */
20
21 /*! \file bmsparsevec_serial.h
22 \brief Serialization for sparse_vector<>
23 */
24
25
26 #ifndef BM__H__INCLUDED__
27 // BitMagic utility headers do not include main "bm.h" declaration
28 // #include "bm.h" or "bm64.h" explicitly
29 # error missing include (bm.h or bm64.h)
30 #endif
31
32 #include "bmsparsevec.h"
33 #include "bmserial.h"
34 #include "bmbuffer.h"
35 #include "bmdef.h"
36
37 namespace bm
38 {
39
40 /** \defgroup svserial Sparse vector serialization
41 Sparse vector serialization
42 \ingroup svector
43 */
44
45
46 /*!
47 \brief layout class for serialization buffer structure
48
49 Class keeps a memory block sized for the target sparse vector BLOB.
50 This class also provides acess to bit-plane memory, so it becomes possible
51 to use parallel storage methods to save bit-plains into
52 different storage shards.
53
54 \ingroup svserial
55 */
56 template<class SV>
57 struct sparse_vector_serial_layout
58 {
59 typedef typename SV::value_type value_type;
60 typedef typename SV::bvector_type bvector_type;
61 typedef typename serializer<bvector_type>::buffer buffer_type;
62
63 sparse_vector_serial_layout() {}
64
65 ~sparse_vector_serial_layout() {}
66
67 /*!
68 \brief resize capacity
69 \param capacity - new capacity
70 \return new buffer or 0 if failed
71 */
72 unsigned char* reserve(size_t capacity)
73 {
74 if (capacity == 0)
75 {
76 freemem();
77 return 0;
78 }
79 buf_.reinit(capacity);
80 return buf_.data();
81 }
82
83 /// return current serialized size
84 size_t size() const { return buf_.size(); }
85
86 /// Set new serialized size
87 void resize(size_t ssize) { buf_.resize(ssize); }
88
89 /// return serialization buffer capacity
90 size_t capacity() const { return buf_.capacity(); }
91
92 /// free memory
93 void freemem() { buf_.release(); }
94
95 /// Set plain output pointer and size
96 void set_plain(unsigned i, unsigned char* ptr, size_t buf_size)
97 {
98 plain_ptrs_[i] = ptr;
99 plane_size_[i] = buf_size;
100 }
101
102 /// Get plain pointer
103 const unsigned char* get_plain(unsigned i) const { return plain_ptrs_[i]; }
104
105 /// Return serialization buffer pointer
106 const unsigned char* buf() const { return buf_.buf(); }
107 /// Return serialization buffer pointer
108 const unsigned char* data() const { return buf_.buf(); }
109
110 private:
111 sparse_vector_serial_layout(const sparse_vector_serial_layout&);
112 void operator=(const sparse_vector_serial_layout&);
113 protected:
114 buffer_type buf_; ///< serialization buffer
115 unsigned char* plain_ptrs_[SV::sv_plains]; ///< pointers on serialized bit-plains
116 size_t plane_size_[SV::sv_plains]; ///< serialized plain size
117 };
118
119 // -------------------------------------------------------------------------
120
121 /*!
122 \brief Serialize sparse vector into a memory buffer(s) structure
123
124 Serialization format:
125
126 | HEADER | BIT-VECTORS ... | REMAP_MATRIX
127
128 Header structure:
129 -----------------
130 BYTE+BYTE: Magic-signature 'BM' or 'BC' (c-compressed)
131 BYTE : Byte order ( 0 - Big Endian, 1 - Little Endian)
132 {
133 BYTE : Number of Bit-vector plains (total) (non-zero when < 255 plains)
134 |
135 BYTE: zero - flag of large plain matrix
136 INT64: Nnmber of bit-vector plains
137 }
138 INT64: Vector size
139 INT64: Offset of plain 0 from the header start (value 0 means plain is empty)
140 INT64: Offset of plain 1 from
141 ...
142 INT32: reserved
143
144 Bit-vectors:
145 ------------
146 Based on current bit-vector serialization
147
148 Remap Matrix:
149 SubHeader | Matrix BLOB
150
151 sub-header:
152 BYTE: 'R' (remapping) or 'N' (no remapping)
153 N - means no other info is saved on the stream
154 INT64: remap matrix size
155
156 \ingroup svector
157 \ingroup svserial
158 */
159 template<typename SV>
160 class sparse_vector_serializer
161 {
162 public:
163 typedef typename SV::bvector_type bvector_type;
164 typedef const bvector_type* bvector_type_const_ptr;
165 typedef bvector_type* bvector_type_ptr;
166 typedef typename SV::value_type value_type;
167 typedef typename SV::size_type size_type;
168 typedef typename bvector_type::allocator_type::allocator_pool_type
169 allocator_pool_type;
170
171 public:
172 sparse_vector_serializer();
173
174 /**
175 Add skip-markers for faster range deserialization
176
177 @param enable - TRUE searilization will add bookmark codes
178 @param bm_interval - bookmark interval in (number of blocks)
179 (suggested between 4 and 512)
180 smaller interval means more bookmarks added to the skip list thus
181 more increasing the BLOB size
182 */
183 void set_bookmarks(bool enable, unsigned bm_interval = 256)
184 { bvs_.set_bookmarks(enable, bm_interval); }
185
186 /// Turn ON and OFF XOR compression of sparse vectors
187 void set_xor_ref(bool is_enabled) { is_xor_ref_ = is_enabled; }
188
189 /// Get XOR reference compression status (enabled/disabled)
190 bool is_xor_ref() const { return is_xor_ref_; }
191
192 /*!
193 \brief Serialize sparse vector into a memory buffer(s) structure
194
195 \param sv - sparse vector to serialize
196 \param sv_layout - buffer structure to keep the result
197 as defined in bm::serialization_flags
198 */
199 void serialize(const SV& sv,
200 sparse_vector_serial_layout<SV>& sv_layout);
201
202 protected:
203 typedef typename
204 bm::serializer<bvector_type>::bv_ref_vector_type bv_ref_vector_type;
205
206 void build_xor_ref_vector(const SV& sv);
207
208 private:
209 sparse_vector_serializer(const sparse_vector_serializer&) = delete;
210 sparse_vector_serializer& operator=(const sparse_vector_serializer&) = delete;
211 protected:
212 bm::serializer<bvector_type> bvs_;
213
214 bool is_xor_ref_;
215 bv_ref_vector_type bv_ref_;
216 };
217
218 /**
219 sparse vector de-serializer
220
221 */
222 template<typename SV>
223 class sparse_vector_deserializer
224 {
225 public:
226 typedef typename SV::bvector_type bvector_type;
227 typedef const bvector_type* bvector_type_const_ptr;
228 typedef bvector_type* bvector_type_ptr;
229 typedef typename SV::value_type value_type;
230 typedef typename SV::size_type size_type;
231 typedef typename bvector_type::allocator_type::allocator_pool_type allocator_pool_type;
232
233 public:
234 sparse_vector_deserializer();
235 ~sparse_vector_deserializer();
236
237 /*!
238 Deserialize sparse vector
239
240 @param sv - [out] target sparse vector to populate
241 @param buf - source memory pointer
242 */
243 void deserialize(SV& sv, const unsigned char* buf)
244 { idx_range_set_ = false; deserialize_sv(sv, buf, 0); }
245
246 /*!
247 Deserialize sparse vector for the range [from, to]
248
249 @param sv - [out] target sparse vector to populate
250 @param buf - source memory pointer
251 @param from - start vector index for deserialization range
252 @param to - end vector index for deserialization range
253 */
254 void deserialize_range(SV& sv, const unsigned char* buf,
255 size_type from, size_type to);
256
257 void deserialize(SV& sv, const unsigned char* buf,
258 size_type from, size_type to)
259 {
260 deserialize_range(sv, buf, from, to);
261 }
262
263
264 /*!
265 Deserialize sparse vector using address mask vector
266 Address mask defines (by set bits) which vector elements to be extracted
267 from the compressed BLOB
268
269 @param sv - [out] target sparse vector to populate
270 @param buf - source memory pointer
271 @param mask_bv - AND mask bit-vector (address gather vector)
272 */
273 void deserialize(SV& sv,
274 const unsigned char* buf,
275 const bvector_type& mask_bv)
276 { idx_range_set_ = false; deserialize_sv(sv, buf, &mask_bv); }
277
278
279 protected:
280 typedef typename bvector_type::allocator_type alloc_type;
281 typedef typename bm::serializer<bvector_type>::bv_ref_vector_type bv_ref_vector_type;
282
283
284 /// Deserialize header/version and other common info
285 ///
286 /// @return number of bit-plains
287 ///
288 unsigned load_header(bm::decoder& dec, SV& sv, unsigned char& matr_s_ser);
289
290 void deserialize_sv(SV& sv, const unsigned char* buf,
291 const bvector_type* mask_bv);
292
293
294 /// deserialize bit-vector plains
295 void deserialize_plains(SV& sv, unsigned plains,
296 const unsigned char* buf,
297 const bvector_type* mask_bv = 0);
298
299 /// load offset table
300 void load_plains_off_table(bm::decoder& dec, unsigned plains);
301
302 /// load NULL bit-plain (returns new plains count)
303 int load_null_plain(SV& sv,
304 int plains,
305 const unsigned char* buf,
306 const bvector_type* mask_bv);
307
308 /// load string remap dict
309 static void load_remap(SV& sv, const unsigned char* remap_buf_ptr);
310
311 /// throw error on incorrect deserialization
312 static void raise_invalid_header();
313 /// throw error on incorrect deserialization
314 static void raise_invalid_64bit();
315 /// throw error on incorrect deserialization
316 static void raise_invalid_bitdepth();
317
318 private:
319 sparse_vector_deserializer(const sparse_vector_deserializer&) = delete;
320 sparse_vector_deserializer& operator=(const sparse_vector_deserializer&) = delete;
321 protected:
322 const unsigned char* remap_buf_ptr_;
323 alloc_type alloc_;
324 bm::word_t* temp_block_;
325 allocator_pool_type pool_;
326 bm::deserializer<bvector_type, bm::decoder> deserial_;
327 bm::operation_deserializer<bvector_type> op_deserial_;
328 bm::rank_compressor<bvector_type> rsc_compressor_;
329 bvector_type not_null_mask_bv_;
330 bvector_type rsc_mask_bv_;
331 bm::heap_vector<size_t, alloc_type, true> off_vect_;
332
333 bv_ref_vector_type bv_ref_;
334
335 bool idx_range_set_;
336 size_type idx_range_from_;
337 size_type idx_range_to_;
338 };
339
340
341
342 /*!
343 \brief Serialize sparse vector into a memory buffer(s) structure
344
345 \param sv - sparse vector to serialize
346 \param sv_layout - buffer structure to keep the result
347 \param temp_block - temporary buffer
348 (allocate with BM_DECLARE_TEMP_BLOCK(x) for speed)
349
350 \ingroup svserial
351
352 @sa serialization_flags
353 @sa sparse_vector_deserializer
354 */
355 template<class SV>
356 void sparse_vector_serialize(
357 const SV& sv,
358 sparse_vector_serial_layout<SV>& sv_layout,
359 bm::word_t* temp_block = 0)
360 {
361 (void)temp_block;
362 bm::sparse_vector_serializer<SV> sv_serializer;
363 sv_serializer.serialize(sv, sv_layout);
364 }
365
366 // -------------------------------------------------------------------------
367
368 /*!
369 \brief Deserialize sparse vector
370 \param sv - target sparse vector
371 \param buf - source memory buffer
372 \param temp_block - temporary block buffer to avoid re-allocations
373
374 \return 0 (error processing via std::logic_error)
375
376 \ingroup svserial
377 @sa sparse_vector_deserializer
378 */
379 template<class SV>
380 int sparse_vector_deserialize(SV& sv,
381 const unsigned char* buf,
382 bm::word_t* temp_block=0)
383 {
384 (void)temp_block;
385 bm::sparse_vector_deserializer<SV> sv_deserializer;
386 sv_deserializer.deserialize(sv, buf);
387 return 0;
388 }
389
390 // -------------------------------------------------------------------------
391
392 /**
393 Seriaizer for compressed collections
394 */
395 template<class CBC>
396 class compressed_collection_serializer
397 {
398 public:
399 typedef CBC compressed_collection_type;
400 typedef typename CBC::bvector_type bvector_type;
401 typedef typename CBC::buffer_type buffer_type;
402 typedef typename CBC::statistics statistics_type;
403 typedef typename CBC::address_resolver_type address_resolver_type;
404
405 public:
406 void serialize(const CBC& buffer_coll,
407 buffer_type& buf,
408 bm::word_t* temp_block = 0);
409 };
410
411 /**
412 Deseriaizer for compressed collections
413 */
414 template<class CBC>
415 class compressed_collection_deserializer
416 {
417 public:
418 typedef CBC compressed_collection_type;
419 typedef typename CBC::bvector_type bvector_type;
420 typedef typename CBC::buffer_type buffer_type;
421 typedef typename CBC::statistics statistics_type;
422 typedef typename CBC::address_resolver_type address_resolver_type;
423 typedef typename CBC::container_type container_type;
424
425 public:
426 int deserialize(CBC& buffer_coll,
427 const unsigned char* buf,
428 bm::word_t* temp_block=0);
429 };
430
431
432 // -------------------------------------------------------------------------
433
434 /**
435 \brief Serialize compressed collection into memory buffer
436
437 Serialization format:
438
439
440 <pre>
441 | MAGIC_HEADER | ADDRESS_BITVECTROR | LIST_OF_BUFFER_SIZES | BUFFER(s)
442
443 MAGIC_HEADER:
444 BYTE+BYTE: Magic-signature 'BM' or 'BC'
445 BYTE : Byte order ( 0 - Big Endian, 1 - Little Endian)
446
447 ADDRESS_BITVECTROR:
448 INT64: address bit-vector size
449 [memblock]: serialized address bit-vector
450
451 LIST_OF_BUFFER_SIZES:
452 INT64 - buffer sizes count
453 INT32 - buffer size 0
454 INT32 - buffer size 1
455 ...
456
457 BUFFERS:
458 [memblock]: block0
459 [memblock]: block1
460 ...
461
462 </pre>
463 */
464
465 template<class CBC>
466 void compressed_collection_serializer<CBC>::serialize(const CBC& buffer_coll,
467 buffer_type& buf,
468 bm::word_t* temp_block)
469 {
470 statistics_type st;
471 buffer_coll.calc_stat(&st);
472
473 buf.resize(st.max_serialize_mem);
474
475 // ptr where bit-plains start
476 unsigned char* buf_ptr = buf.data();
477
478 bm::encoder enc(buf.data(), buf.capacity());
479 ByteOrder bo = globals<true>::byte_order();
480 enc.put_8('B');
481 enc.put_8('C');
482 enc.put_8((unsigned char)bo);
483
484 unsigned char* mbuf1 = enc.get_pos(); // bookmark position
485 enc.put_64(0); // address vector size (reservation)
486
487 buf_ptr = enc.get_pos();
488
489 const address_resolver_type& addr_res = buffer_coll.resolver();
490 const bvector_type& bv = addr_res.get_bvector();
491 {
492 bm::serializer<bvector_type > bvs(temp_block);
493 bvs.gap_length_serialization(false);
494
495 size_t addr_bv_size = bvs.serialize(bv, buf_ptr, buf.size());
496 buf_ptr += addr_bv_size;
497
498 enc.set_pos(mbuf1); // rewind to bookmark
499 enc.put_64(addr_bv_size); // save the address vector size
500 }
501 enc.set_pos(buf_ptr); // restore stream position
502 size_t coll_size = buffer_coll.size();
503
504 enc.put_64(coll_size);
505
506 // pass 1 (save buffer sizes)
507 {
508 for (unsigned i = 0; i < buffer_coll.size(); ++i)
509 {
510 const buffer_type& cbuf = buffer_coll.get(i);
511 size_t sz = cbuf.size();
512 enc.put_64(sz);
513 } // for i
514 }
515 // pass 2 (save buffers)
516 {
517 for (unsigned i = 0; i < buffer_coll.size(); ++i)
518 {
519 const buffer_type& cbuf = buffer_coll.get(i);
520 size_t sz = cbuf.size();
521 enc.memcpy(cbuf.buf(), sz);
522 } // for i
523 }
524 buf.resize(enc.size());
525 }
526
527 // -------------------------------------------------------------------------
528 template<class CBC>
529 int compressed_collection_deserializer<CBC>::deserialize(
530 CBC& buffer_coll,
531 const unsigned char* buf,
532 bm::word_t* temp_block)
533 {
534 // TODO: implement correct processing of byte-order corect deserialization
535 // ByteOrder bo_current = globals<true>::byte_order();
536
537 bm::decoder dec(buf);
538 unsigned char h1 = dec.get_8();
539 unsigned char h2 = dec.get_8();
540
541 BM_ASSERT(h1 == 'B' && h2 == 'C');
542 if (h1 != 'B' && h2 != 'C') // no magic header? issue...
543 {
544 return -1;
545 }
546 //unsigned char bv_bo =
547 dec.get_8();
548
549 // -----------------------------------------
550 // restore address resolver
551 //
552 bm::id64_t addr_bv_size = dec.get_64();
553
554 const unsigned char* bv_buf_ptr = dec.get_pos();
555
556 address_resolver_type& addr_res = buffer_coll.resolver();
557 bvector_type& bv = addr_res.get_bvector();
558 bv.clear();
559
560 bm::deserialize(bv, bv_buf_ptr, temp_block);
561 addr_res.sync();
562
563 typename bvector_type::size_type addr_cnt = bv.count();
564 dec.seek((int)addr_bv_size);
565
566 // -----------------------------------------
567 // read buffer sizes
568 //
569 bm::id64_t coll_size = dec.get_64();
570 if (coll_size != addr_cnt)
571 {
572 return -2; // buffer size collection does not match address vector
573 }
574
575 typedef std::vector<unsigned>::size_type vect_size_type;
576 std::vector<bm::id64_t> buf_size_vec;
577 buf_size_vec.resize(vect_size_type(coll_size));
578 {
579 for (unsigned i = 0; i < coll_size; ++i)
580 {
581 bm::id64_t sz = dec.get_64();
582 buf_size_vec[i] = sz;
583 } // for i
584 }
585
586 {
587 container_type& buf_vect = buffer_coll.container();
588 buf_vect.resize(vect_size_type(coll_size));
589 for (unsigned i = 0; i < coll_size; ++i)
590 {
591 bm::id64_t sz = buf_size_vec[i];
592 buffer_type& b = buf_vect.at(i);
593 b.resize(sz);
594 dec.memcpy(b.data(), sz);
595 } // for i
596 }
597 buffer_coll.sync();
598 return 0;
599 }
600
601 // -------------------------------------------------------------------------
602 //
603 // -------------------------------------------------------------------------
604
605 template<typename SV>
606 sparse_vector_serializer<SV>::sparse_vector_serializer()
607 {
608 bvs_.gap_length_serialization(false);
609 #ifdef BMXORCOMP
610 is_xor_ref_ = true;
611 #else
612 is_xor_ref_ = false;
613 #endif
614 }
615
616 // -------------------------------------------------------------------------
617
618 template<typename SV>
619 void sparse_vector_serializer<SV>::build_xor_ref_vector(const SV& sv)
620 {
621 bv_ref_.reset();
622 bv_ref_.build(sv.get_bmatrix());
623 }
624
625 // -------------------------------------------------------------------------
626
627 template<typename SV>
628 void sparse_vector_serializer<SV>::serialize(const SV& sv,
629 sparse_vector_serial_layout<SV>& sv_layout)
630 {
631 typename SV::statistics sv_stat;
632 sv.calc_stat(&sv_stat);
633 unsigned char* buf = sv_layout.reserve(sv_stat.max_serialize_mem);
634
635 bm::encoder enc(buf, (unsigned)sv_layout.capacity());
636 unsigned plains = sv.stored_plains();
637
638 // header size in bytes
639 unsigned h_size = 1 + 1 + // "BM" or "BC" (magic header)
640 1 + // byte-order
641 1 + // number of bit-plains (for vector)
642 8 + // size (internal 64-bit)
643 (8 * plains) + // offsets of all plains
644 4; // reserve
645 // for large plain matrixes
646 {
647 h_size += 1 + // version number
648 8; // number of plains (64-bit)
649 }
650
651 // ---------------------------------
652 // Setup XOR reference compression
653 //
654 if (is_xor_ref())
655 {
656 build_xor_ref_vector(sv);
657 bvs_.set_ref_vectors(&bv_ref_);
658 }
659
660 // -----------------------------------------------------
661 // Serialize all bvector plains
662 //
663
664 unsigned char* buf_ptr = buf + h_size; // ptr where plains start (start+hdr)
665
666 unsigned i;
667 for (i = 0; i < plains; ++i)
668 {
669 typename SV::bvector_type_const_ptr bv = sv.get_plain(i);
670 if (!bv) // empty plain
671 {
672 sv_layout.set_plain(i, 0, 0);
673 continue;
674 }
675 if (is_xor_ref())
676 {
677 unsigned idx = (unsigned)bv_ref_.find(i);
678 BM_ASSERT(idx != bv_ref_.not_found());
679 bvs_.set_curr_ref_idx(idx);
680 }
681
682 size_t buf_size =
683 bvs_.serialize(*bv, buf_ptr, sv_stat.max_serialize_mem);
684
685 sv_layout.set_plain(i, buf_ptr, buf_size);
686 buf_ptr += buf_size;
687 if (sv_stat.max_serialize_mem > buf_size)
688 {
689 sv_stat.max_serialize_mem -= buf_size;
690 continue;
691 }
692 BM_ASSERT(0); // TODO: throw an exception here
693 } // for i
694
695 bvs_.set_ref_vectors(0); // disangage XOR ref vector
696
697 // -----------------------------------------------------
698 // serialize the re-map matrix
699 //
700 if (bm::conditional<SV::is_remap_support::value>::test()) // test remapping trait
701 {
702 bm::encoder enc_m(buf_ptr, sv_stat.max_serialize_mem);
703 if (sv.is_remap())
704 {
705 bm::id64_t remap_size = sv.remap_size();
706 const unsigned char* matrix_buf = sv.get_remap_buffer();
707 BM_ASSERT(matrix_buf);
708 BM_ASSERT(remap_size);
709
710 enc_m.put_8('R');
711 enc_m.put_64(remap_size);
712 enc_m.memcpy(matrix_buf, remap_size);
713 enc_m.put_8('E'); // end of matrix (integrity check token)
714 }
715 else
716 {
717 enc_m.put_8('N');
718 }
719 buf_ptr += enc_m.size(); // add mattrix encoded data size
720 }
721
722 sv_layout.resize(size_t(buf_ptr - buf)); // set the true occupied size
723
724 // -----------------------------------------------------
725 // save the header
726 //
727 ByteOrder bo = bm::globals<true>::byte_order();
728
729 enc.put_8('B'); // magic header 'BM' - bit matrix 'BC' - bit compressed
730 if (sv.is_compressed())
731 enc.put_8('C');
732 else
733 enc.put_8('M');
734
735 enc.put_8((unsigned char)bo); // byte order
736
737 unsigned char matr_s_ser = 1;
738 #ifdef BM64ADDR
739 matr_s_ser = 2;
740 #endif
741
742 enc.put_8(0); // number of plains == 0 (legacy magic number)
743 enc.put_8(matr_s_ser); // matrix serialization version
744 enc.put_64(plains); // number of rows in the bit-matrix
745 enc.put_64(sv.size_internal());
746
747 // save the offset table (part of the header)
748 //
749 for (i = 0; i < plains; ++i)
750 {
751 const unsigned char* p = sv_layout.get_plain(i);
752 if (!p)
753 {
754 enc.put_64(0);
755 continue;
756 }
757 size_t offset = size_t(p - buf);
758 enc.put_64(offset);
759 } // for
760 }
761
762 // -------------------------------------------------------------------------
763 //
764 // -------------------------------------------------------------------------
765
766 template<typename SV>
767 sparse_vector_deserializer<SV>::sparse_vector_deserializer()
768 : remap_buf_ptr_(0)
769 {
770 temp_block_ = alloc_.alloc_bit_block();
771 not_null_mask_bv_.set_allocator_pool(&pool_);
772 rsc_mask_bv_.set_allocator_pool(&pool_);
773 idx_range_set_ = false;
774 }
775
776 // -------------------------------------------------------------------------
777
778 template<typename SV>
779 sparse_vector_deserializer<SV>::~sparse_vector_deserializer()
780 {
781 if (temp_block_)
782 alloc_.free_bit_block(temp_block_);
783 }
784
785 // -------------------------------------------------------------------------
786
787 template<typename SV>
788 void sparse_vector_deserializer<SV>::deserialize_range(SV& sv,
789 const unsigned char* buf,
790 size_type from, size_type to)
791 {
792 idx_range_set_ = true; idx_range_from_ = from; idx_range_to_ = to;
793
794 remap_buf_ptr_ = 0;
795 bm::decoder dec(buf); // TODO: implement correct processing of byte-order
796
797 unsigned char matr_s_ser = 0;
798 unsigned plains = load_header(dec, sv, matr_s_ser);
799
800 sv.clear();
801
802 bm::id64_t sv_size = dec.get_64();
803 if (sv_size == 0)
804 return; // empty vector
805
806 sv.resize_internal(size_type(sv_size));
807 bv_ref_.reset();
808
809 load_plains_off_table(dec, plains); // read the offset vector of bit-plains
810
811 // TODO: add range for not NULL plane
812 plains = (unsigned)load_null_plain(sv, int(plains), buf, 0);
813
814 // check if mask needs to be relaculated using the NULL (index) vector
815 if (bm::conditional<SV::is_rsc_support::value>::test())
816 {
817 // recalculate plains range
818 size_type sv_left, sv_right;
819 bool range_valid = sv.resolve_range(from, to, &sv_left, &sv_right);
820 if (!range_valid)
821 {
822 sv.clear();
823 idx_range_set_ = false;
824 return;
825 }
826 else
827 {
828 idx_range_set_ = true; idx_range_from_ = sv_left; idx_range_to_ = sv_right;
829 }
830 }
831
832 deserialize_plains(sv, plains, buf, 0);
833
834 op_deserial_.set_ref_vectors(0);
835 deserial_.set_ref_vectors(0);
836 bv_ref_.reset();
837
838
839 // load the remap matrix
840 //
841 if (bm::conditional<SV::is_remap_support::value>::test()) // test remap trait
842 {
843 if (matr_s_ser)
844 load_remap(sv, remap_buf_ptr_);
845 } // if remap traits
846
847 sv.sync(true); // force sync, recalculate RS index, remap tables, etc
848 remap_buf_ptr_ = 0;
849
850 idx_range_set_ = false;
851 }
852
853 // -------------------------------------------------------------------------
854
855 template<typename SV>
856 void sparse_vector_deserializer<SV>::deserialize_sv(SV& sv,
857 const unsigned char* buf,
858 const bvector_type* mask_bv)
859 {
860 remap_buf_ptr_ = 0;
861 bm::decoder dec(buf); // TODO: implement correct processing of byte-order
862
863 unsigned char matr_s_ser = 0;
864 unsigned plains = load_header(dec, sv, matr_s_ser);
865
866 sv.clear();
867
868 bm::id64_t sv_size = dec.get_64();
869 if (sv_size == 0)
870 return; // empty vector
871
872 sv.resize_internal(size_type(sv_size));
873 bv_ref_.reset();
874
875 load_plains_off_table(dec, plains); // read the offset vector of bit-plains
876
877 plains = (unsigned)load_null_plain(sv, int(plains), buf, mask_bv);
878
879 // check if mask needs to be relaculated using the NULL (index) vector
880 if (bm::conditional<SV::is_rsc_support::value>::test())
881 {
882 if (mask_bv)
883 {
884 const bvector_type* bv_null = sv.get_null_bvector();
885 BM_ASSERT(bv_null);
886 rsc_mask_bv_.clear(true);
887 not_null_mask_bv_.bit_and(*bv_null, *mask_bv, bvector_type::opt_compress);
888 rsc_compressor_.compress(rsc_mask_bv_, *bv_null, not_null_mask_bv_);
889 mask_bv = &rsc_mask_bv_;
890
891 // if it needs range recalculation
892 if (idx_range_set_) // range setting is in effect
893 {
894 //bool rf =
895 rsc_mask_bv_.find_range(idx_range_from_, idx_range_to_);
896 }
897 }
898 }
899
900 deserialize_plains(sv, plains, buf, mask_bv);
901
902 op_deserial_.set_ref_vectors(0);
903 deserial_.set_ref_vectors(0);
904 bv_ref_.reset();
905
906
907 // load the remap matrix
908 //
909 if (bm::conditional<SV::is_remap_support::value>::test()) // test remap trait
910 {
911 if (matr_s_ser)
912 load_remap(sv, remap_buf_ptr_);
913 } // if remap traits
914
915 sv.sync(true); // force sync, recalculate RS index, remap tables, etc
916 remap_buf_ptr_ = 0;
917 }
918
919 // -------------------------------------------------------------------------
920
921 template<typename SV>
922 unsigned sparse_vector_deserializer<SV>::load_header(
923 bm::decoder& dec, SV& sv, unsigned char& matr_s_ser)
924 {
925 unsigned char h1 = dec.get_8();
926 unsigned char h2 = dec.get_8();
927
928 BM_ASSERT(h1 == 'B' && (h2 == 'M' || h2 == 'C'));
929
930 bool sig2_ok = (h2 == 'M' || h2 == 'C');
931 if (h1 != 'B' || !sig2_ok) //&& (h2 != 'M' || h2 != 'C')) // no magic header?
932 raise_invalid_header();
933
934 unsigned char bv_bo = dec.get_8(); (void) bv_bo;
935 unsigned plains = dec.get_8();
936 if (plains == 0) // bit-matrix
937 {
938 matr_s_ser = dec.get_8(); // matrix serialization version
939 plains = (unsigned) dec.get_64(); // number of rows in the bit-matrix
940 }
941 #ifdef BM64ADDR
942 #else
943 if (matr_s_ser == 2) // 64-bit matrix
944 {
945 raise_invalid_64bit();
946 }
947 #endif
948
949 unsigned sv_plains = sv.stored_plains();
950 if (!plains || plains > sv_plains)
951 raise_invalid_bitdepth();
952 return plains;
953 }
954
955 // -------------------------------------------------------------------------
956
957 template<typename SV>
958 void sparse_vector_deserializer<SV>::deserialize_plains(
959 SV& sv,
960 unsigned plains,
961 const unsigned char* buf,
962 const bvector_type* mask_bv)
963 {
964 if (mask_bv && !idx_range_set_)
965 idx_range_set_ = mask_bv->find_range(idx_range_from_, idx_range_to_);
966
967 op_deserial_.set_ref_vectors(&bv_ref_);
968 deserial_.set_ref_vectors(&bv_ref_);
969
970 // read-deserialize the plains based on offsets
971 // backward order to bring the NULL vector first
972 //
973 for (int i = int(plains-1); i >= 0; --i)
974 {
975 size_t offset = off_vect_[unsigned(i)];
976 if (!offset) // empty vector
977 continue;
978 const unsigned char* bv_buf_ptr = buf + offset; // seek to position
979 bvector_type* bv = sv.get_plain(unsigned(i));
980 BM_ASSERT(bv);
981
982 bv_ref_.add(bv, unsigned(i));
983
984 if (mask_bv) // gather mask set, use AND operation deserializer
985 {
986 typename bvector_type::mem_pool_guard mp_g_z(pool_, *bv);
987
988 if (bm::conditional<SV::is_remap_support::value>::test()
989 && !remap_buf_ptr_) // last plain vector (special case)
990 {
991 size_t read_bytes =
992 deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
993 remap_buf_ptr_ = bv_buf_ptr + read_bytes;
994 bv->bit_and(*mask_bv, bvector_type::opt_compress);
995 continue;
996 }
997 if (idx_range_set_)
998 deserial_.set_range(idx_range_from_, idx_range_to_);
999 deserial_.deserialize(*bv, bv_buf_ptr);
1000 bv->bit_and(*mask_bv, bvector_type::opt_compress);
1001 }
1002 else
1003 {
1004 if (bm::conditional<SV::is_remap_support::value>::test() &&
1005 !remap_buf_ptr_)
1006 {
1007 size_t read_bytes =
1008 deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1009 remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1010 if (idx_range_set_)
1011 bv->keep_range(idx_range_from_, idx_range_to_);
1012 continue;
1013 }
1014 if (idx_range_set_)
1015 {
1016 deserial_.set_range(idx_range_from_, idx_range_to_);
1017 deserial_.deserialize(*bv, bv_buf_ptr);
1018 bv->keep_range(idx_range_from_, idx_range_to_);
1019 }
1020 else
1021 {
1022 //size_t read_bytes =
1023 deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1024 }
1025 }
1026
1027 } // for i
1028 deserial_.unset_range();
1029
1030 }
1031
1032 // -------------------------------------------------------------------------
1033
1034 template<typename SV>
1035 int sparse_vector_deserializer<SV>::load_null_plain(SV& sv,
1036 int plains,
1037 const unsigned char* buf,
1038 const bvector_type* mask_bv)
1039 {
1040 BM_ASSERT(plains > 0);
1041 if (!sv.is_nullable())
1042 return plains;
1043 int i = plains - 1;
1044 size_t offset = off_vect_[unsigned(i)];
1045 if (offset)
1046 {
1047 // TODO: improve serialization format to avoid non-range decode of
1048 // the NULL vector just to get to the offset of remap table
1049
1050 const unsigned char* bv_buf_ptr = buf + offset; // seek to position
1051 bvector_type* bv = sv.get_plain(unsigned(i));
1052 bv_ref_.add(bv, unsigned(i));
1053 if (bm::conditional<SV::is_rsc_support::value>::test())
1054 {
1055 // load the whole not-NULL vector regardless of range
1056 // TODO: load [0, idx_range_to_]
1057 size_t read_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1058 remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1059 }
1060 else // non-compressed SV
1061 {
1062 // NULL plain in string vector with substitute re-map
1063 //
1064 if (bm::conditional<SV::is_remap_support::value>::test())
1065 {
1066 BM_ASSERT(!remap_buf_ptr_);
1067 size_t read_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1068 remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1069 if (idx_range_set_)
1070 bv->keep_range(idx_range_from_, idx_range_to_);
1071 }
1072 else
1073 if (idx_range_set_)
1074 {
1075 deserial_.set_range(idx_range_from_, idx_range_to_);
1076 deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1077 bv->keep_range(idx_range_from_, idx_range_to_);
1078 deserial_.unset_range();
1079 }
1080 else
1081 {
1082 deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1083 }
1084 if (mask_bv)
1085 bv->bit_and(*mask_bv, bvector_type::opt_compress);
1086 }
1087 }
1088 return plains-1;
1089 }
1090
1091 // -------------------------------------------------------------------------
1092
1093 template<typename SV>
1094 void sparse_vector_deserializer<SV>::load_plains_off_table(
1095 bm::decoder& dec, unsigned plains)
1096 {
1097 off_vect_.resize(plains);
1098 for (unsigned i = 0; i < plains; ++i)
1099 {
1100 size_t offset = (size_t) dec.get_64();
1101 off_vect_[i] = offset;
1102 } // for i
1103 }
1104
1105 // -------------------------------------------------------------------------
1106
1107 template<typename SV>
1108 void sparse_vector_deserializer<SV>::load_remap(SV& sv,
1109 const unsigned char* remap_buf_ptr)
1110 {
1111 if (!remap_buf_ptr)
1112 return;
1113
1114 bm::decoder dec_m(remap_buf_ptr);
1115 unsigned char rh = dec_m.get_8();
1116 switch (rh)
1117 {
1118 case 'N':
1119 break;
1120 case 'R':
1121 {
1122 size_t remap_size = (size_t) dec_m.get_64();
1123 unsigned char* remap_buf = sv.init_remap_buffer();
1124 BM_ASSERT(remap_buf);
1125 size_t target_remap_size = sv.remap_size();
1126 if (!remap_size || !remap_buf || remap_size != target_remap_size)
1127 {
1128 #ifndef BM_NO_STL
1129 throw std::logic_error("Invalid serialization format (remap size)");
1130 #else
1131 BM_THROW(BM_ERR_SERIALFORMAT);
1132 #endif
1133 }
1134 dec_m.memcpy(remap_buf, remap_size);
1135 unsigned char end_tok = dec_m.get_8();
1136 if (end_tok != 'E')
1137 {
1138 #ifndef BM_NO_STL
1139 throw std::logic_error("Invalid serialization format");
1140 #else
1141 BM_THROW(BM_ERR_SERIALFORMAT);
1142 #endif
1143 }
1144 sv.set_remap();
1145 }
1146 break;
1147 default:
1148 #ifndef BM_NO_STL
1149 throw std::logic_error("Invalid serialization format (remap error)");
1150 #else
1151 BM_THROW(BM_ERR_SERIALFORMAT);
1152 #endif
1153 } // switch
1154 }
1155
1156 // -------------------------------------------------------------------------
1157
1158 template<typename SV>
1159 void sparse_vector_deserializer<SV>::raise_invalid_header()
1160 {
1161 #ifndef BM_NO_STL
1162 throw std::logic_error("Invalid serialization signature header");
1163 #else
1164 BM_THROW(BM_ERR_SERIALFORMAT);
1165 #endif
1166 }
1167
1168 // -------------------------------------------------------------------------
1169
1170 template<typename SV>
1171 void sparse_vector_deserializer<SV>::raise_invalid_64bit()
1172 {
1173 #ifndef BM_NO_STL
1174 throw std::logic_error("Invalid serialization target (64-bit BLOB)");
1175 #else
1176 BM_THROW(BM_ERR_SERIALFORMAT);
1177 #endif
1178 }
1179
1180 // -------------------------------------------------------------------------
1181
1182 template<typename SV>
1183 void sparse_vector_deserializer<SV>::raise_invalid_bitdepth()
1184 {
1185 #ifndef BM_NO_STL
1186 throw std::logic_error("Invalid serialization target (bit depth)");
1187 #else
1188 BM_THROW(BM_ERR_SERIALFORMAT);
1189 #endif
1190 }
1191
1192 // -------------------------------------------------------------------------
1193
1194 } // namespace bm
1195
1196 #include "bmundef.h"
1197
1198 #endif
bm::compressed_collection_serializer::buffer_type
CBC::buffer_type buffer_type
Definition: bmsparsevec_serial.h:401
bm::compressed_collection_serializer::address_resolver_type
CBC::address_resolver_type address_resolver_type
Definition: bmsparsevec_serial.h:403
bm::deserialize
size_t deserialize(BV &bv, const unsigned char *buf, bm::word_t *temp_block=0, const bm::bv_ref_vector< BV > *ref_vect=0)
Bitvector deserialization from a memory BLOB.
Definition: bmserial.h:2688
bm::sparse_vector_deserializer::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:229
bm::bvector::mem_pool_guard
Definition: bm.h:771
bm::sparse_vector_serial_layout::plane_size_
size_t plane_size_[SV::sv_plains]
serialized plain size
Definition: bmsparsevec_serial.h:116
bm::sparse_vector_deserializer::raise_invalid_bitdepth
static void raise_invalid_bitdepth()
throw error on incorrect deserialization
Definition: bmsparsevec_serial.h:1183
bm::sparse_vector_deserializer::load_header
unsigned load_header(bm::decoder &dec, SV &sv, unsigned char &matr_s_ser)
Deserialize header/version and other common info.
Definition: bmsparsevec_serial.h:922
bm::sparse_vector_serializer::bv_ref_
bv_ref_vector_type bv_ref_
Definition: bmsparsevec_serial.h:215
bm::sparse_vector_deserializer::load_plains_off_table
void load_plains_off_table(bm::decoder &dec, unsigned plains)
load offset table
Definition: bmsparsevec_serial.h:1094
bm::sparse_vector_deserializer
sparse vector de-serializer
Definition: bmsparsevec_serial.h:223
bm::decoder_base::get_pos
const unsigned char * get_pos() const BMNOEXCEPT
Return current buffer pointer.
Definition: encoding.h:101
bm::sparse_vector_deserializer::load_remap
static void load_remap(SV &sv, const unsigned char *remap_buf_ptr)
load string remap dict
Definition: bmsparsevec_serial.h:1108
bm::sparse_vector_deserializer::alloc_
alloc_type alloc_
Definition: bmsparsevec_serial.h:323
bm::sparse_vector_serial_layout::buf_
buffer_type buf_
serialization buffer
Definition: bmsparsevec_serial.h:114
bm::sparse_vector_serializer::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:163
bm::encoder::put_64
void put_64(bm::id64_t w) BMNOEXCEPT
Puts 64 bits word into encoding buffer.
Definition: encoding.h:562
bm::sparse_vector_deserializer::not_null_mask_bv_
bvector_type not_null_mask_bv_
Definition: bmsparsevec_serial.h:329
bm::sparse_vector_serial_layout::buf
const unsigned char * buf() const
Return serialization buffer pointer.
Definition: bmsparsevec_serial.h:106
bm::sparse_vector_serializer::is_xor_ref
bool is_xor_ref() const
Get XOR reference compression status (enabled/disabled)
Definition: bmsparsevec_serial.h:190
bm::sparse_vector_serializer::allocator_pool_type
bvector_type::allocator_type::allocator_pool_type allocator_pool_type
Definition: bmsparsevec_serial.h:169
bm::id64_t
unsigned long long int id64_t
Definition: bmconst.h:34
bmsparsevec.h
Sparse constainer sparse_vector<> for integer types using bit-transposition transform.
bm::serializer::serialize
size_type serialize(const BV &bv, unsigned char *buf, size_t buf_size)
Bitvector serialization into memory block.
Definition: bmserial.h:2264
bm::encoder::size
size_t size() const BMNOEXCEPT
Returns size of the current encoding stream.
Definition: encoding.h:485
bm::sparse_vector_serial_layout::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:60
bm::sparse_vector_deserializer::bv_ref_
bv_ref_vector_type bv_ref_
Definition: bmsparsevec_serial.h:333
bm::sparse_vector_serializer::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:166
bm::compressed_collection_deserializer::container_type
CBC::container_type container_type
Definition: bmsparsevec_serial.h:423
bm::sparse_vector_deserializer::alloc_type
bvector_type::allocator_type alloc_type
Definition: bmsparsevec_serial.h:280
bm::compressed_collection_deserializer::buffer_type
CBC::buffer_type buffer_type
Definition: bmsparsevec_serial.h:420
bm::decoder_base::memcpy
void memcpy(unsigned char *dst, size_t count) BMNOEXCEPT
read bytes from the decode buffer
Definition: encoding.h:618
bm::bvector::allocator_type
Alloc allocator_type
Definition: bm.h:110
bm::sparse_vector_serializer::is_xor_ref_
bool is_xor_ref_
Definition: bmsparsevec_serial.h:214
bm::sparse_vector_deserializer::sparse_vector_deserializer
sparse_vector_deserializer()
Definition: bmsparsevec_serial.h:767
bm::compressed_collection_deserializer::deserialize
int deserialize(CBC &buffer_coll, const unsigned char *buf, bm::word_t *temp_block=0)
Definition: bmsparsevec_serial.h:529
bm::sparse_vector_deserialize
int sparse_vector_deserialize(SV &sv, const unsigned char *buf, bm::word_t *temp_block=0)
Deserialize sparse vector.
Definition: bmsparsevec_serial.h:380
bm::sparse_vector_deserializer::bvector_type_ptr
bvector_type * bvector_type_ptr
Definition: bmsparsevec_serial.h:228
bm::sparse_vector_serialize
void sparse_vector_serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout, bm::word_t *temp_block=0)
Serialize sparse vector into a memory buffer(s) structure.
Definition: bmsparsevec_serial.h:356
bm::decoder_base::get_8
unsigned char get_8() BMNOEXCEPT
Reads character from the decoding buffer.
Definition: encoding.h:89
bm::compressed_collection_deserializer::bvector_type
CBC::bvector_type bvector_type
Definition: bmsparsevec_serial.h:419
bm::sparse_vector_deserializer::deserialize
void deserialize(SV &sv, const unsigned char *buf, const bvector_type &mask_bv)
Definition: bmsparsevec_serial.h:273
bm::sparse_vector_serial_layout
layout class for serialization buffer structure
Definition: bmsparsevec_serial.h:57
bmundef.h
pre-processor un-defines to avoid global space pollution (internal)
bm::globals::byte_order
static ByteOrder byte_order()
Definition: bmconst.h:464
bm::sparse_vector_serial_layout::sparse_vector_serial_layout
sparse_vector_serial_layout()
Definition: bmsparsevec_serial.h:63
bm::compressed_collection_serializer
Seriaizer for compressed collections.
Definition: bmsparsevec_serial.h:396
bm::encoder::put_8
void put_8(unsigned char c) BMNOEXCEPT
Puts one character into the encoding buffer.
Definition: encoding.h:420
bm::sparse_vector_serial_layout::~sparse_vector_serial_layout
~sparse_vector_serial_layout()
Definition: bmsparsevec_serial.h:65
bm::sparse_vector_serial_layout::capacity
size_t capacity() const
return serialization buffer capacity
Definition: bmsparsevec_serial.h:90
bm::decoder::get_64
bm::id64_t get_64() BMNOEXCEPT
Reads 64-bit word from the decoding buffer.
Definition: encoding.h:703
bm::encoder::set_pos
void set_pos(unsigned char *buf_pos) BMNOEXCEPT
Set current memory stream position.
Definition: encoding.h:501
bm::compressed_collection_serializer::compressed_collection_type
CBC compressed_collection_type
Definition: bmsparsevec_serial.h:399
bm::sparse_vector_deserializer::deserial_
bm::deserializer< bvector_type, bm::decoder > deserial_
Definition: bmsparsevec_serial.h:326
bm::sparse_vector_deserializer::idx_range_from_
size_type idx_range_from_
Definition: bmsparsevec_serial.h:336
bm::sparse_vector_serial_layout::reserve
unsigned char * reserve(size_t capacity)
resize capacity
Definition: bmsparsevec_serial.h:72
bm::bvector::allocator_pool_type
allocator_type::allocator_pool_type allocator_pool_type
Definition: bm.h:111
bm::rank_compressor< bvector_type >
bm::sparse_vector_deserializer::bv_ref_vector_type
bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type
Definition: bmsparsevec_serial.h:281
bm::sparse_vector_deserializer::pool_
allocator_pool_type pool_
Definition: bmsparsevec_serial.h:325
bm::serializer::gap_length_serialization
void gap_length_serialization(bool value) BMNOEXCEPT
Set GAP length serialization (serializes GAP levels of the original vector)
Definition: bmserial.h:1126
bm::sparse_vector_deserializer::off_vect_
bm::heap_vector< size_t, alloc_type, true > off_vect_
Definition: bmsparsevec_serial.h:331
bm::sparse_vector_deserializer::idx_range_to_
size_type idx_range_to_
Definition: bmsparsevec_serial.h:337
bm::bvector::opt_compress
@ opt_compress
compress blocks when possible (GAP/prefix sum)
Definition: bm.h:134
bm::compressed_collection_serializer::bvector_type
CBC::bvector_type bvector_type
Definition: bmsparsevec_serial.h:400
bm::sparse_vector_deserializer::size_type
SV::size_type size_type
Definition: bmsparsevec_serial.h:230
bm::sparse_vector_serializer::serialize
void serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout)
Serialize sparse vector into a memory buffer(s) structure.
Definition: bmsparsevec_serial.h:628
bm::sparse_vector_deserializer::allocator_pool_type
bvector_type::allocator_type::allocator_pool_type allocator_pool_type
Definition: bmsparsevec_serial.h:231
bm::sparse_vector_deserializer::~sparse_vector_deserializer
~sparse_vector_deserializer()
Definition: bmsparsevec_serial.h:779
bm::sparse_vector_serializer::bv_ref_vector_type
bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type
Definition: bmsparsevec_serial.h:204
bm::sparse_vector_serial_layout::buffer_type
serializer< bvector_type >::buffer buffer_type
Definition: bmsparsevec_serial.h:61
bm::conditional
ad-hoc conditional expressions
Definition: bmutil.h:110
bm::sparse_vector_deserializer::idx_range_set_
bool idx_range_set_
Definition: bmsparsevec_serial.h:335
BM_ASSERT
#define BM_ASSERT
Definition: bmdef.h:130
bm::sparse_vector_serial_layout::resize
void resize(size_t ssize)
Set new serialized size.
Definition: bmsparsevec_serial.h:87
bvector_type
bm::bvector bvector_type
Definition: strsvsample01.cpp:39
bm::sparse_vector_deserializer::deserialize_sv
void deserialize_sv(SV &sv, const unsigned char *buf, const bvector_type *mask_bv)
Definition: bmsparsevec_serial.h:856
bm::deserializer< bvector_type, bm::decoder >
bm::operation_deserializer< bvector_type >
bm::sparse_vector_deserializer::deserialize
void deserialize(SV &sv, const unsigned char *buf)
Definition: bmsparsevec_serial.h:243
bmdef.h
Definitions(internal)
bm::compressed_collection_serializer::statistics_type
CBC::statistics statistics_type
Definition: bmsparsevec_serial.h:402
bm::sparse_vector_serializer::bvector_type_const_ptr
const typedef bvector_type * bvector_type_const_ptr
Definition: bmsparsevec_serial.h:164
bm::serializer
Bit-vector serialization class.
Definition: bmserial.h:75
bm::sparse_vector_serializer::set_bookmarks
void set_bookmarks(bool enable, unsigned bm_interval=256)
Add skip-markers for faster range deserialization.
Definition: bmsparsevec_serial.h:183
bm::compressed_collection_deserializer
Deseriaizer for compressed collections.
Definition: bmsparsevec_serial.h:415
bm::compressed_collection_serializer::serialize
void serialize(const CBC &buffer_coll, buffer_type &buf, bm::word_t *temp_block=0)
Definition: bmsparsevec_serial.h:466
bm::sparse_vector_deserializer::temp_block_
bm::word_t * temp_block_
Definition: bmsparsevec_serial.h:324
bm::sparse_vector_deserializer::bvector_type_const_ptr
const typedef bvector_type * bvector_type_const_ptr
Definition: bmsparsevec_serial.h:227
bm::sparse_vector_serial_layout::set_plain
void set_plain(unsigned i, unsigned char *ptr, size_t buf_size)
Set plain output pointer and size.
Definition: bmsparsevec_serial.h:96
bm::compressed_collection_deserializer::address_resolver_type
CBC::address_resolver_type address_resolver_type
Definition: bmsparsevec_serial.h:422
bm::sparse_vector_serializer::bvs_
bm::serializer< bvector_type > bvs_
Definition: bmsparsevec_serial.h:212
bm::sparse_vector_deserializer::deserialize_plains
void deserialize_plains(SV &sv, unsigned plains, const unsigned char *buf, const bvector_type *mask_bv=0)
deserialize bit-vector plains
Definition: bmsparsevec_serial.h:958
bm::ByteOrder
ByteOrder
Byte orders recognized by the library.
Definition: bmconst.h:428
bm::sparse_vector_serial_layout::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:59
bmserial.h
Serialization / compression of bvector<>. Set theoretical operations on compressed BLOBs.
bm::compressed_collection_deserializer::compressed_collection_type
CBC compressed_collection_type
Definition: bmsparsevec_serial.h:418
bm::sparse_vector_serial_layout::size
size_t size() const
return current serialized size
Definition: bmsparsevec_serial.h:84
bm::sparse_vector_deserializer::rsc_mask_bv_
bvector_type rsc_mask_bv_
Definition: bmsparsevec_serial.h:330
bm::sparse_vector_deserializer::raise_invalid_64bit
static void raise_invalid_64bit()
throw error on incorrect deserialization
Definition: bmsparsevec_serial.h:1171
bm::sparse_vector_serializer::bvector_type_ptr
bvector_type * bvector_type_ptr
Definition: bmsparsevec_serial.h:165
bm
Definition: bm.h:76
bm::encoder::get_pos
unsigned char * get_pos() const BMNOEXCEPT
Get current memory stream position.
Definition: encoding.h:493
bm::sparse_vector_serial_layout::get_plain
const unsigned char * get_plain(unsigned i) const
Get plain pointer.
Definition: bmsparsevec_serial.h:103
bm::sparse_vector_deserializer::load_null_plain
int load_null_plain(SV &sv, int plains, const unsigned char *buf, const bvector_type *mask_bv)
load NULL bit-plain (returns new plains count)
Definition: bmsparsevec_serial.h:1035
bm::sparse_vector_serial_layout::plain_ptrs_
unsigned char * plain_ptrs_[SV::sv_plains]
pointers on serialized bit-plains
Definition: bmsparsevec_serial.h:115
bm::sparse_vector_deserializer::deserialize
void deserialize(SV &sv, const unsigned char *buf, size_type from, size_type to)
Definition: bmsparsevec_serial.h:257
bm::sparse_vector_serial_layout::data
const unsigned char * data() const
Return serialization buffer pointer.
Definition: bmsparsevec_serial.h:108
bm::sparse_vector_deserializer::deserialize_range
void deserialize_range(SV &sv, const unsigned char *buf, size_type from, size_type to)
Definition: bmsparsevec_serial.h:788
bm::sparse_vector_deserializer::remap_buf_ptr_
const unsigned char * remap_buf_ptr_
Definition: bmsparsevec_serial.h:322
bm::word_t
unsigned int word_t
Definition: bmconst.h:38
bm::sparse_vector_serializer::build_xor_ref_vector
void build_xor_ref_vector(const SV &sv)
Definition: bmsparsevec_serial.h:619
bm::bvector::size_type
bm::id_t size_type
Definition: bm.h:117
bm::serializer::set_bookmarks
void set_bookmarks(bool enable, unsigned bm_interval=256) BMNOEXCEPT
Add skip-markers to serialization BLOB for faster range decode at the expense of some BLOB size incre...
Definition: bmserial.h:1138
bm::sparse_vector_serializer::set_xor_ref
void set_xor_ref(bool is_enabled)
Turn ON and OFF XOR compression of sparse vectors.
Definition: bmsparsevec_serial.h:187
bm::encoder
Memory encoding.
Definition: encoding.h:49
bm::sparse_vector_deserializer::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:226
bm::sparse_vector_serializer::sparse_vector_serializer
sparse_vector_serializer()
Definition: bmsparsevec_serial.h:606
bm::decoder
Class for decoding data from memory buffer.
Definition: encoding.h:117
bm::sparse_vector_serial_layout::freemem
void freemem()
free memory
Definition: bmsparsevec_serial.h:93
bm::sparse_vector_serializer
Definition: bmsparsevec_serial.h:160
bm::decoder_base::seek
void seek(int delta) BMNOEXCEPT
change current position
Definition: encoding.h:95
bm::sparse_vector_deserializer::raise_invalid_header
static void raise_invalid_header()
throw error on incorrect deserialization
Definition: bmsparsevec_serial.h:1159
bm::compressed_collection_deserializer::statistics_type
CBC::statistics statistics_type
Definition: bmsparsevec_serial.h:421
bm::encoder::memcpy
void memcpy(const unsigned char *src, size_t count) BMNOEXCEPT
copy bytes into target buffer or just rewind if src is NULL
Definition: encoding.h:472
bm::sparse_vector_deserializer::op_deserial_
bm::operation_deserializer< bvector_type > op_deserial_
Definition: bmsparsevec_serial.h:327
bm::bv_ref_vector
List of reference bit-vectors with their true index associations.
Definition: bmxor.h:240
bm::sparse_vector_deserializer::rsc_compressor_
bm::rank_compressor< bvector_type > rsc_compressor_
Definition: bmsparsevec_serial.h:328
bm::sparse_vector_serializer::size_type
SV::size_type size_type
Definition: bmsparsevec_serial.h:167