Utf8.hpp

Go to the documentation of this file.

/*
 *
 * ---
 *
 *  Copyright (C) 2020 Anselm Schmidt (ans[ät]ohai.su)
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version in addition to the terms of any
 *  licences already herein identified.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 *
 * ---
 *
 * Utf8.hpp
 *
 * Namespace for global UTF-8 helper functions.
 *
 *  Created on: Dec 10, 2018
 *      Author: ans
 */

#ifndef HELPER_UTF8_HPP_
#define HELPER_UTF8_HPP_

#include "../Main/Exception.hpp"

#include "../_extern/utf8/source/utf8.h"

#include <cstddef>      // std::size_t
#include <string>       // std::string
#include <string_view>  // std::string_view

namespace crawlservpp::Helper::Utf8 {

    /*
     * CONSTANTS
     */


    inline constexpr auto utf8MemoryFactor{2};

    inline constexpr auto bitmaskTopBit{0x80};

    inline constexpr auto bitmaskTopTwoBits{0xc0};

    inline constexpr auto shiftSixBits{6};

    inline constexpr auto bitmaskLastSixBits0b000001{0x3F};

    inline constexpr auto oneByte{1};

    inline constexpr auto twoBytes{2};

    inline constexpr auto threeBytes{3};

    inline constexpr auto fourBytes{4};


    /*
     * DECLARATION
     */


    std::string iso88591ToUtf8(std::string_view strIn);


    bool isValidUtf8(std::string_view stringToCheck, std::string& errTo);
    bool isLastCharValidUtf8(std::string_view stringToCheck);
    bool isSingleUtf8Char(std::string_view stringToCheck);


    bool repairUtf8(std::string_view strIn, std::string& strOut);


    std::size_t length(std::string_view str);


    /*
     * EXCEPTION CLASS
     */


    MAIN_EXCEPTION_CLASS();

    /*
     * IMPLEMENTATION
     */

    /*
     * CONVERSION
     */


    inline std::string iso88591ToUtf8(std::string_view strIn) {
        std::string strOut;

        // guess maximum memory requirement
        strOut.reserve(strIn.size() * utf8MemoryFactor);

        for(const uint8_t c : strIn) {
            if(c < bitmaskTopBit) {
                strOut.push_back(c);
            }
            else {
                strOut.push_back(bitmaskTopTwoBits | c >> shiftSixBits);
                strOut.push_back(bitmaskTopBit | (c & bitmaskLastSixBits0b000001));
            }
        }

        return strOut;
    }

    /*
     * VALIDATION
     */


    inline bool isValidUtf8(std::string_view stringToCheck, std::string& errTo) {
        try {
            return utf8::is_valid(stringToCheck.cbegin(), stringToCheck.cend());
        }
        catch(const utf8::exception& e) {
            errTo = e.what();

            return false;
        }
    }


    inline bool isLastCharValidUtf8(const std::string& stringToCheck) {
        if(stringToCheck.empty()) {
            return true;
        }

        // check for valid one-byte character
        auto pos{stringToCheck.size() - 1};

        if(utf8::is_valid(stringToCheck.substr(pos, oneByte))) {
            return true;
        }

        if(stringToCheck.size() < twoBytes) {
            return false;
        }

        --pos;

        // check for valid two-byte character
        if(utf8::is_valid(stringToCheck.substr(pos, twoBytes))) {
            return true;
        }

        if(stringToCheck.size() < threeBytes) {
            return false;
        }

        --pos;

        // check for valid three-byte character
        if(utf8::is_valid(stringToCheck.substr(pos, threeBytes))) {
            return true;
        }

        if(stringToCheck.size() < fourBytes) {
            return false;
        }

        --pos;

        // check for valid four-byte character
        if(utf8::is_valid(stringToCheck.substr(pos, fourBytes))) {
            return true;
        }

        return false;
    }


    inline bool isSingleUtf8Char(std::string_view stringToCheck) {
        return utf8::distance(stringToCheck.begin(), stringToCheck.end()) == 1;
    }

    /*
     * REPAIR
     */


    inline bool repairUtf8(std::string_view strIn, std::string& strOut) {
        try {
            if(utf8::is_valid(strIn.cbegin(), strIn.cend())) {
                return false;
            }

            utf8::replace_invalid(strIn.begin(), strIn.end(), back_inserter(strOut));

            return true;
        }
        catch(const utf8::exception& e) {
            throw Exception("UTF-8 error: " + std::string(e.what()));
        }
    }

    /*
     * LENGTH
     */

    inline std::size_t length(std::string_view str) {
        constexpr unsigned char maxOneByte{127};
        constexpr unsigned char checkTwoBytes{0xE0};
        constexpr unsigned char isTwoBytes{0xC0};
        constexpr unsigned char checkThreeBytes{0xF0};
        constexpr unsigned char isThreeBytes{0xE0};
        constexpr unsigned char checkFourBytes{0xF8};
        constexpr unsigned char isFourBytes{0xF0};

        constexpr std::size_t skipTwoBytes{2};
        constexpr std::size_t skipThreeBytes{3};

        std::size_t result{};

        const auto bytes{
            str.length()
        };

        for(std::size_t pos{}; pos < bytes; ++pos) {
            ++result;

            const unsigned char byte{
                static_cast<unsigned char>(str[pos])
            };

            if(byte <= maxOneByte) {
                // one byte
                continue;
            }

            if((byte & checkTwoBytes) == isTwoBytes) {
                // two bytes
                ++pos;
            }
            else if((byte & checkThreeBytes) == isThreeBytes) {
                // three bytes
                pos += skipTwoBytes;
            }
            else if((byte & checkFourBytes) == isFourBytes) {
                // four bytes
                pos += skipThreeBytes;
            }
            else {
                std::string exceptionString{"Invalid UTF-8 in '"};

                exceptionString += str;
                exceptionString += "'";

                throw Exception(exceptionString);
            }
        }

        return result;
    }

} /* namespace crawlservpp::Helper::Utf8 */

#endif /* HELPER_UTF8_HPP_ */