ciff_lin.h

Go to the documentation of this file.

/*
    CIFF_LIN.H
    ----------
    Copyright (c) 2019 Andrew Trotman
    Released under the 2-clause BSD license (See:https://en.wikipedia.org/wiki/BSD_licenses)
*/
#pragma once

#include <stdint.h>

#include <vector>
#include <limits>

#include "posting.h"
#include "protobuf.h"

namespace JASS
    {
    /*
        CLASS CIFF_LIN
        --------------
    */
    class ciff_lin
        {
        public:
            enum error_code
                {
                OK = 0,         
                FAIL = 1            
                } ;

            /*
                CLASS CIFF_LIN::HEADER
                ----------------------
            */
            class header
                {
                public:
                    int32_t version;                        
                    int32_t num_postings_lists;             
                    int32_t num_docs;                       
                    int32_t total_postings_lists;           
                    int32_t total_docs;                     
                    int64_t total_terms_in_collection;  
                    double average_doclength;               
                    std::string description;                        

                public:
                    /*
                        CIFF_LIN::HEADER::HEADER()
                        --------------------------
                    */
                    header():
                        version(0),
                        num_postings_lists(0),
                        num_docs(0),
                        total_postings_lists(0),
                        total_docs(0),
                        total_terms_in_collection(0),
                        average_doclength(0),
                        description("")
                        {
                        /* Nothing */
                        }

                    /*
                        CIFF_LIN::HEADER::TEXT_RENDER()
                        -------------------------------
                    */
                    void text_render(void)
                        {
                        std::cout << "version:" << version << "\n";
                        std::cout << "num_postings_lists:" << num_postings_lists << "\n";
                        std::cout << "num_docs:" << num_docs << "\n";
                        std::cout << "total_postings_lists:" << total_postings_lists << "\n";
                        std::cout << "total_docs:" << total_docs << "\n";
                        std::cout << "total_terms_in_collection:" << total_terms_in_collection << "\n";
                        std::cout << "average_doclength:" << average_doclength << "\n";
                        std::cout << "description:" << description << "\n";
                        }
                };

            /*
                CLASS CIFF_LIN::POSTINGS_LIST
                -----------------------------
            */
            class postings_list
                {
                public:
                    slice term;                                 
                    uint64_t document_frequency;            
                    uint64_t collection_frequency;      
                    std::vector<posting> postings;      

                private:
                    /*
                        CIFF_LIN::POSTINGS_LIST::GET_NEXT_POSTINGS_PAIR()
                        -------------------------------------------------
                    */
                    static error_code get_next_postings_pair(posting &into, const uint8_t *&stream, const uint8_t *stream_end)
                        {
                        while (stream < stream_end)
                            {
                            protobuf::wire_type type;
                            uint8_t field = protobuf::get_type_and_field(type, stream);

                            if (type == protobuf::VARINT)
                                {
                                uint64_t value = protobuf::get_uint64_t(stream);
                                if (field == 1)
                                    into.docid = static_cast<uint32_t>(value);
                                else if (field == 2)
                                    into.term_frequency = static_cast<uint32_t>(value);
                                else
                                    return FAIL;
                                }
                            else
                                return FAIL;
                            }
                        return OK;
                        }

                public:
                    /*
                        CIFF_LIN::POSTINGS_LIST::GET_NEXT_POSTINGS()
                        --------------------------------------------
                    */
                    static error_code get_next_postings(postings_list &into, const uint8_t *&stream, const uint8_t *stream_end)
                        {
                        while (stream < stream_end)
                            {
                            protobuf::wire_type type;
                            uint8_t field = protobuf::get_type_and_field(type, stream);

                            if (type == protobuf::VARINT)
                                {
                                uint64_t value = protobuf::get_uint64_t(stream);

                                if (field == 2)
                                    into.document_frequency = value;
                                else if (field == 3)
                                    into.collection_frequency = value;
                                else
                                    return FAIL;
                                }
                            else if (type == protobuf::BLOB)
                                {
                                slice term = protobuf::get_blob(stream);

                                if (field == 1)
                                    into.term = term;
                                else if (field == 4)
                                    {
                                    const uint8_t *here = reinterpret_cast<const uint8_t *> (term.address());
                                    posting d_tf_pair;
                                    if (get_next_postings_pair(d_tf_pair, here, here + term.size()) == FAIL)
                                        return FAIL;
                                    into.postings.push_back(d_tf_pair);
                                    }
                                }
                            else
                                return FAIL;
                            }
                        return OK;
                        }

                /*
                    CIFF_LIN::POSTINGS_LIST::CLEAR()
                    --------------------------------
                */
                void clear(void)
                    {
                    term.clear();
                    document_frequency = 0;
                    collection_frequency = 0;
                    postings.clear();
                    }
                };

            /*
                CLASS CIFF_LIN::DOC_RECORD
                --------------------------
            */
            class doc_record
                {
                public:
                    int32_t docid;                  
                    slice collection_docid;     
                    int32_t doclength;          

                public:
                    /*
                        CIFF_LIN::DOC_RECORD::DOC_RECORD()
                        ----------------------------------
                    */
                    doc_record() :
                        docid(0),
                        doclength(0)
                        {
                        /* Nothing */
                        }

                    /*
                        CIFF_LIN::DOC_RECORD::CLEAR()
                        -----------------------------
                    */
                    void clear(void)
                        {
                        docid = 0;
                        collection_docid.clear();
                        doclength = 0;
                        }

                    /*
                        CIFF_LIN::DOC_RECORD::GET_NEXT()
                        --------------------------------
                    */
                    static error_code get_next(doc_record &into, const uint8_t *&stream)
                        {
                        size_t length = protobuf::get_uint64_t(stream);
                        const uint8_t *stream_end = stream + length;

                        while (stream < stream_end)
                            {
                            protobuf::wire_type type;
                            uint8_t field = protobuf::get_type_and_field(type, stream);

                            if (type == protobuf::VARINT)
                                {
                                uint64_t value = protobuf::get_uint64_t(stream);

                                if (field == 1)
                                    into.docid = (uint32_t)value;
                                else if (field == 3)
                                    into.doclength = (uint32_t)value;
                                else
                                    return FAIL;
                                }
                            else if (type == protobuf::BLOB)
                                {
                                slice term = protobuf::get_blob(stream);
                                if (field == 2)
                                    into.collection_docid = term;
                                else
                                    return FAIL;
                                }
                            else
                                return FAIL;
                            }
                        return OK;
                        }
                };
                
        private:
            /*
                CLASS CIFF_LIN::POSTINGS_LIST_ITERATOR
                --------------------------------------
            */
            class postings_list_iterator
                {
                private:
                    ciff_lin &source;                   
                    const uint8_t *stream;          
                    size_t which;                       
                    postings_list postings;         

                public:
                    /*
                        CIFF_LIN::POSTINGS_LIST_ITERATOR::POSTINGS_LIST_ITERATOR()
                        ----------------------------------------------------------
                    */
                    postings_list_iterator(ciff_lin &of, size_t which) :
                        source(of),
                        stream(source.stream),
                        which(which)
                        {
                        /* Nothing */
                        }

                    /*
                        CIFF_LIN::POSTINGS_LIST_ITERATOR::OPERATOR++()
                        ----------------------------------------------
                    */
                    postings_list_iterator &operator++()
                        {
                        which++;
                        return *this;
                        }

                    /*
                        CIFF_LIN::POSTINGS_LIST_ITERATOR::OPERATOR*()
                        ---------------------------------------------
                    */
                    postings_list &operator*()
                        {
                        int64_t postings_list_length = protobuf::get_uint64_t(stream);

                        postings.clear();
                        if (postings_list::get_next_postings(postings, stream, stream + postings_list_length) == FAIL)
                            {
                            /*
                                On error move to the end of the stream and mark the stream as bad
                            */
                            which = (std::numeric_limits<decltype(which)>::max)();
                            source.status = FAIL;
                            }

                        return postings;
                        }

                    /*
                        CIFF_LIN::POSTINGS_LIST_ITERATOR::OPERATOR!=()
                        ----------------------------------------------
                    */
                    bool operator!=(postings_list_iterator &with)
                        {
                        bool answer = which <= with.which;
                        source.stream = stream;
                        return answer;
                        }
                };

            /*
                CLASS CIFF_LIN::POSTINGS_FOREACH
                --------------------------------
            */
            class postings_foreach
                {
                private:
                    ciff_lin &parent;

                public:
                    postings_foreach(ciff_lin &object) :
                        parent(object)
                        {
                        /* Nothing */
                        }

                /*
                    CIFF_LIN::POSTINGS_FOREACH::BEGIN()
                    -----------------------------------
                */
                ciff_lin::postings_list_iterator begin()
                    {
                    return ++ciff_lin::postings_list_iterator(parent, 0);
                    }

                /*
                    CIFF_LIN::POSTINGS_FOREACH::END()
                    ---------------------------------
                */
                ciff_lin::postings_list_iterator end()
                    {
                    return ciff_lin::postings_list_iterator(parent, parent.ciff_header.num_postings_lists);
                    }
                };

            /*
                CLASS CIFF_LIN::DOCRECORDS_ITERATOR
                -----------------------------------
            */
            class docrecords_iterator
                {
                private:
                    ciff_lin &source;                   
                    const uint8_t *stream;          
                    size_t which;                       
                    doc_record current;             

                public:

                    /*
                        CIFF_LIN::DOCRECORDS_ITERATOR::DOCRECORDS_ITERATOR()
                        ----------------------------------------------------
                    */
                    docrecords_iterator(ciff_lin &of, size_t which) :
                        source(of),
                        stream(source.stream),
                        which(which)
                        {
                        /* Nothing */
                        }

                    /*
                        CIFF_LIN::DOCRECORDS_ITERATOR::OPERATOR++()
                        -------------------------------------------
                    */
                    docrecords_iterator &operator++()
                        {
                        current.clear();

                        if (doc_record::get_next(current, stream) == FAIL)
                            {
                            /*
                                On error move to the end of the stream and mark the stream as bad
                            */
                            which = (std::numeric_limits<decltype(which)>::max)();
                            source.status = FAIL;
                            }

                        which++;
                        return *this;
                        }

                    /*
                        CIFF_LIN::DOCRECORDS_ITERATOR::OPERATOR*()
                        ------------------------------------------
                    */
                    doc_record &operator*()
                        {
                        return current;
                        }

                    /*
                        CIFF_LIN::DOCRECORDS_ITERATOR::OPERATOR!=()
                        -------------------------------------------
                    */
                    bool operator!=(docrecords_iterator &with)
                        {
                        return which <= with.which;
                        }
                };

            /*
                CLASS CIFF_LIN::DOCRECORDS_FOREACH
                ----------------------------------
            */
            class docrecords_foreach
                {
                private:
                    ciff_lin &parent;

                public:
                    docrecords_foreach(ciff_lin &object) :
                        parent(object)
                        {
                        /* Nothing */
                        }

                /*
                    CIFF_LIN::DOCRECORDS_FOREACH::BEGIN()
                    -------------------------------------
                */
                ciff_lin::docrecords_iterator begin()
                    {
                    return ++ciff_lin::docrecords_iterator(parent, 0);
                    }

                /*
                    CIFF_LIN::DOCRECORDS_FOREACH::END()
                    -----------------------------------
                */
                ciff_lin::docrecords_iterator end()
                    {
                    return ciff_lin::docrecords_iterator(parent, parent.ciff_header.num_docs);
                    }
                };

        private:
            const uint8_t *source_file;             
            const uint8_t *stream;                      
            header ciff_header;                         
            
        public:
            error_code status;                          

        protected:
            /*
                READ_HEADER()
                -------------
            */
            error_code read_header(header &header)
                {
                stream = source_file;
                size_t length = protobuf::get_uint64_t(stream);
                const uint8_t *stream_end = stream + length;

                while (stream < stream_end)
                    {
                    protobuf::wire_type type;
                    uint8_t field = protobuf::get_type_and_field(type, stream);

                    if (type == protobuf::VARINT)
                        {
                        uint64_t value = protobuf::get_uint64_t(stream);

                        if (field == 1)
                            header.version = (uint32_t)value;
                        else if (field == 2)
                            header.num_postings_lists = (uint32_t)value;
                        else if (field == 3)
                            header.num_docs = (uint32_t)value;
                        else if (field == 4)
                            header.total_postings_lists = (uint32_t)value;
                        else if (field == 5)
                            header.total_docs = (uint32_t)value;
                        else if (field == 6)
                            header.total_terms_in_collection = value;
                        else
                            return FAIL;
                        }
                    else if (type == protobuf::SIXTY_FOUR_BIT)
                        {
                        double value = protobuf::get_double(stream);
                        if (field == 7)
                            header.average_doclength = value;
                        else
                            return FAIL;
                        }
                    else if (type == protobuf::BLOB)
                        {
                        slice term = protobuf::get_blob(stream);

                        if (field == 8)
                            header.description = std::string(reinterpret_cast<char *>(term.address()), term.size());
                        else
                            return FAIL;
                        }
                    else
                        return FAIL;
                    }
                return OK;
                }

        public:
            /*
                CIFF_LIN::CIFF_LIN()
                --------------------
            */
            ciff_lin(const uint8_t *source_file) :
                source_file(source_file),
                stream(source_file),
                status(OK)
                {
                read_header(ciff_header);
                ciff_header.text_render();
                }

            /*
                CIFF_LIN::POSTINGS()
                --------------------
            */
            postings_foreach postings(void)
                {
                return postings_foreach(*this);
                }

            /*
                CIFF_LIN::DOCRECORDS()
                ----------------------
            */
            docrecords_foreach docrecords(void)
                {
                return docrecords_foreach(*this);
                }

            /*
                CIFF_LIN::GET_HEADER()
                ----------------------
            */
            header &get_header(void)
                {
                return ciff_header;
                }
        } ;
    }