crawlserv++
[under development]
Application for crawling and analyzing textual content of websites.
|
#include "../../Main/Exception.hpp"
#include "../../Module/Config.hpp"
#include <algorithm>
#include <cstdint>
#include <string>
#include <vector>
Go to the source code of this file.
Classes | |
class | crawlservpp::Module::Parser::Config |
Configuration for parsers. More... | |
struct | crawlservpp::Module::Parser::Config::Entries |
Configuration entries for parser threads. More... | |
class | crawlservpp::Module::Parser::Config::Exception |
Class for parser configuration exceptions. More... | |
Namespaces | |
crawlservpp::Module::Parser | |
Namespace for parser classes. | |
Constants | |
constexpr std::uint8_t | crawlservpp::Module::Parser::crawlerLoggingVerbose {0} |
Logging is disabled. More... | |
constexpr std::uint8_t | crawlservpp::Module::Parser::generalLoggingDefault {1} |
Default logging is enabled. More... | |
constexpr std::uint8_t | crawlservpp::Module::Parser::generalLoggingExtended {2} |
Extended logging is enabled. More... | |
constexpr std::uint8_t | crawlservpp::Module::Parser::generalLoggingVerbose {3} |
Verbose logging is enabled. More... | |
constexpr std::uint8_t | crawlservpp::Module::Parser::parsingSourceUrl {0} |
Parse data from the URL of a crawled web page. More... | |
constexpr std::uint8_t | crawlservpp::Module::Parser::parsingSourceContent {1} |
Parse data from the content of a crawled web page. More... | |
constexpr std::uint64_t | crawlservpp::Module::Parser::defaultCacheSize {2500} |
Default cache size. More... | |
constexpr std::uint32_t | crawlservpp::Module::Parser::defaultLockS {300} |
Default URL locking time, in seconds. More... | |
constexpr std::uint16_t | crawlservpp::Module::Parser::defaultMaxBatchSize {500} |
Default maximum number of URLs to be processed in one MySQL query. More... | |
constexpr std::uint64_t | crawlservpp::Module::Parser::defaultSleepIdleMs {5000} |
Default time to wait before checking for new URLs when all URLs have been parsed, in milliseconds. More... | |
constexpr std::uint64_t | crawlservpp::Module::Parser::defaultSleepMySqlS {60} |
Default time to wait before last try to re-connect to MySQL server, in seconds. More... | |