serialise_jass_v1.h

Go to the documentation of this file.

/*
    SERIALISE_JASS_V1.H
    -------------------
    Copyright (c) 2016 Andrew Trotman
    Released under the 2-clause BSD license (See:https://en.wikipedia.org/wiki/BSD_licenses)
*/
#pragma once

#include "file.h"
#include "slice.h"
#include "allocator_cpp.h"
#include "index_postings.h"
#include "index_manager.h"
#include "compress_integer_qmx_jass_v1.h"
#include "compress_integer_elias_gamma_simd.h"

namespace JASS
    {
    /*
        CLASS SERIALISE_JASS_V1
        -----------------------
    */
    class serialise_jass_v1 : public index_manager::delegate
        {
        protected:
            /*
                CLASS SERIALISE_JASS_V1::VOCAB_TRIPPLE
                --------------------------------------
            */
            class vocab_tripple
                {
                public:
                    slice   token;              
                    uint64_t term;              
                    uint64_t offset;            
                    uint64_t impacts;           

                public:
                    /*
                         SERIALISE_JASS_V1::VOCAB_TRIPPLE::VOCAB_TRIPPLE()
                        --------------------------------------------------
                    */
                    vocab_tripple(const slice &string, uint64_t term, uint64_t offset, uint64_t impacts) :
                        token(string),
                        term(term),
                        offset(offset),
                        impacts(impacts)
                        {
                        /* Nothing */
                        }
                    
                    /*
                         SERIALISE_JASS_V1::VOCAB_TRIPPLE::OPERATOR<()
                        ----------------------------------------------
                    */
                    bool operator<(const vocab_tripple &other) const
                        {
                        return slice::strict_weak_order_less_than(token, other.token);
                        }
                };

        public:
            /*
                ENUM JASS_V1_CODEX
                ------------------
            */
            enum  jass_v1_codex
                {
                uncompressed = 's',             
                variable_byte = 'c',                
                simple_8b = '8',                    
                qmx = 'q',                          
                qmx_d4 = 'Q',                       
                qmx_d0 = 'R',                       
                elias_gamma_simd = 'G',         
                elias_gamma_simd_vb = 'g',      
                elias_delta_simd = 'D'          
                };

        protected:
            file vocabulary_strings;                            
            file vocabulary;                                        
            file postings;                                          
            file primary_keys;                                  
            std::vector<vocab_tripple> index_key;           
            std::vector<uint64_t> primary_key_offsets;  
            allocator_pool memory;                              
            index_postings_impact impact_ordered;           
            std::string compressor_name;                        
            int compressor_d_ness;                              
            compress_integer *encoder;                          
            allocator_cpp<uint8_t> allocator;               
            std::vector<uint8_t, allocator_cpp<uint8_t>> compressed_buffer;     
            std::vector<slice, allocator_cpp<slice>> compressed_segments;           
            uint8_t alignment;                                  

        protected:
            /*
                SERIALISE_JASS_V1::WRITE_POSTINGS()
                -----------------------------------
            */
            virtual size_t write_postings(const index_postings &postings, size_t &number_of_impacts, compress_integer::integer document_frequency, compress_integer::integer *document_ids, index_postings_impact::impact_type *term_frequencies);

        public:
            /*
                SERIALISE_JASS_V1::SERIALISE_JASS_V1()
                --------------------------------------
            */
            serialise_jass_v1(size_t documents, jass_v1_codex codex = jass_v1_codex::elias_gamma_simd, int8_t alignment = 1) :
                index_manager::delegate(documents),
                vocabulary_strings("CIvocab_terms.bin", "w+b"),
                vocabulary("CIvocab.bin", "w+b"),
                postings("CIpostings.bin", "w+b"),
                primary_keys("CIdoclist.bin", "w+b"),
                memory(1024 * 1024),                                
                impact_ordered(documents, memory),
                encoder(get_compressor(codex, compressor_name, compressor_d_ness)),
                allocator(memory),
                compressed_buffer(allocator),
                compressed_segments(allocator),
                alignment(alignment)
                {
                /*
                    Allocate space for storing the compressed postings.  But, allocate too much space as some
                    encoders can't write a sequence smaller than a minimum size.

                    How large does this need to be?
                    We store the:
                        postings, each docid is 8 bytes and there are |documents| of those
                        The impact header consisting of an impact (2 bytes) + start_pointer (8 bytes) + length (8 bytes) + frequency (4 bytes)
                        There are index_postings_impact::largest_impact of those.
                    To make things worse, each of these are stored compressed, and so might be bigger than the
                    raw size by 8/7 (assuming variable byte), so the raw storage is:
                         8/7 *(documents * 8 + 22 * index_postings_impact::largest_impact)
                    but, each of these two things is stored in a vector as a slice and each slice takes
                    8 bytes for the address and 8 bytes for the size giving an additional 2 * 16 * index_postings_impact::largest_impact
                    giving a total of
                        8/7 *(documents * 8 + (22 + 2 * 16) * index_postings_impact::largest_impact)
                    and now lets add a bit for reasons we can't predict (the std::vector has house-keeping)
                        1 MB
                    and make sure integer rounding doesn't get this wrong:
                        8 * (documents * 8 + (22 + 2 * 16) * index_postings_impact::largest_impact) / 7 + 1024 * 1024
                */
                compressed_buffer.resize(8 * (documents * 8 + (22 + 2 * 16) * index_postings_impact::largest_impact) / 7 + 1024 * 1024);
                compressed_segments.reserve(index_postings_impact::largest_impact);

// std::cout << compressor_name << "-D" << compressor_d_ness << "\n";

                postings.write(&codex, 1);
                }

            /*
                SERIALISE_JASS_V1::~SERIALISE_JASS_V1()
                --------------------------------------
            */
            virtual ~serialise_jass_v1()
                {
                /* Nothing */
                }

            /*
                SERIALISE_JASS_V1::FINISH()
                ---------------------------
            */
            virtual void finish(void);

            /*
                 SERIALISE_JASS_V1::SERIALISE_VOCABULARY_POINTERS()
                --------------------------------------------------
            */
            virtual void serialise_vocabulary_pointers(void);

            /*
                 SERIALISE_JASS_V1::SERIALISE_PRIMARY_KEYS()
                --------------------------------------------
            */
            virtual void serialise_primary_keys(void);

            /*
                SERIALISE_JASS_V1::DELEGATE::OPERATOR()()
                -----------------------------------------
            */
            virtual void operator()(const slice &term, const index_postings &postings, compress_integer::integer document_frequency, compress_integer::integer *document_ids, index_postings_impact::impact_type *term_frequencies);

            /*
                SERIALISE_JASS_V1::DELEGATE::OPERATOR()()
                -----------------------------------------
            */
            virtual void operator()(size_t document_id, const slice &primary_key);


            /*
                SERIALISE_JASS_V1::GET_COMPRESSOR()
                -----------------------------------
            */
            static compress_integer *get_compressor(jass_v1_codex codex, std::string &name, int32_t &d_ness);

            /*
                SERIALISE_JASS_V1::UNITTEST()
                -----------------------------
            */
            static void unittest(void);
        };
    }