parser.h

Go to the documentation of this file.

/*
    PARSER.H
    --------
    Copyright (c) 2016 Andrew Trotman
    Released under the 2-clause BSD license (See:https://en.wikipedia.org/wiki/BSD_licenses)
*/
#pragma once

#include <stdint.h>

#include "slice.h"
#include "document.h"
#include "index_postings_impact.h"

namespace JASS
    {
    /*
        CLASS PARSER
        ------------
    */
    class parser
        {
        public:
            /*
                CLASS PARSER::TOKEN
                -------------------
            */
            class token
                {
                public:
                    enum token_type
                        {
                        alpha,                              
                        numeric,                                
                        xml_start_tag,                      
                        xml_empty_tag,                      
                        xml_end_tag,                        
                        xml_comment,                        
                        xml_cdata,                          
                        xml_definition,                 
                        xml_conditional,                    
                        xml_processing_instruction, 
                        other,                              
                        eof                                 
                        };
                public:
                    static constexpr size_t max_token_length = 1024;        

                public:
                    uint8_t buffer[max_token_length];               
                    slice lexeme;                                           
                    token_type type;                                        
                    index_postings_impact::impact_type count;       

                public:
                    /*
                        PARSER::TOKEN::GET()
                        --------------------
                    */
                    const slice &get(void) const
                        {
                        return lexeme;
                        }

                    /*
                        PARSER::TOKEN::SET()
                        --------------------
                    */
                    void set(slice term)
                        {
                        *buffer = '\0';
                        lexeme = term;
                        type = other;
                        count = 1;
                        }
                };

        private:
            document build_document;            

        protected:
            token eof_token;                        
            const document *the_document;       
            const uint8_t *current;             
            const uint8_t *end_of_document; 
            token current_token;                    
            
        protected:
            /*
                PARSER::BUILD_UNICODE_ALPHABETIC_TOKEN()
                ----------------------------------------
            */
            void build_unicode_alphabetic_token(uint32_t codepoint, size_t bytes, uint8_t *&buffer_pos, uint8_t *buffer_end);

            /*
                PARSER::BUILD_UNICODE_NUMERIC_TOKEN()
                -------------------------------------
            */
            void build_unicode_numeric_token(uint32_t codepoint, size_t bytes, uint8_t *&buffer_pos, uint8_t *buffer_end);

        public:
            /*
                PARSER::PARSER()
                ----------------
            */
            parser()
                {
                current = NULL;
                the_document = NULL;
                end_of_document = NULL;
                eof_token.type =  token::eof;
                current_token.count = 1;            // We're generating a stream of individual tokens, so each all counts are 1.
                }
            
            /*
                PARSER::~PARSER()
                -----------------
            */
            virtual ~parser()
                {
                /*
                    Nothing
                */
                }

            /*
                PARSER::SET_DOCUMENT()
                ----------------------
            */
            virtual void set_document(const class document &document)
                {
                the_document = &document;
                current = (uint8_t *)document.contents.address();
                end_of_document = (uint8_t *)document.contents.address() + document.contents.size();
                }

            /*
                PARSER::SET_DOCUMENT()
                ----------------------
            */
            virtual void set_document(const std::string &document)
                {
                current = reinterpret_cast<const uint8_t *>(document.c_str());
                end_of_document = current + document.size();
                }

            /*
                PARSER::GET_NEXT_TOKEN()
                ------------------------
            */
            virtual const class parser::token &get_next_token(void);

            /*
                PARSER::UNITTEST_COUNT()
                ------------------------
            */
            static size_t unittest_count(const char *string);

            /*
                PARSER::UNITTEST()
                ------------------
            */
            static void unittest(void);
        };
    }