|
crawlserv++
[under development]
Application for crawling and analyzing textual content of websites.
|
#include "Config.hpp"#include "../../Data/Corpus.hpp"#include "../../Data/Data.hpp"#include "../../Helper/CommaLocale.hpp"#include "../../Helper/Container.hpp"#include "../../Helper/Json.hpp"#include "../../Helper/Memory.hpp"#include "../../Helper/Portability/mysqlcppconn.h"#include "../../Main/Exception.hpp"#include "../../Struct/CorpusProperties.hpp"#include "../../Struct/StatusSetter.hpp"#include "../../Struct/TableColumn.hpp"#include "../../Struct/TargetTableProperties.hpp"#include "../../Struct/TextMap.hpp"#include "../../Timer/Simple.hpp"#include "../../Wrapper/Database.hpp"#include "../../Wrapper/DatabaseLock.hpp"#include "../../_extern/rapidjson/include/rapidjson/document.h"#include <cppconn/exception.h>#include <cppconn/prepared_statement.h>#include <cppconn/resultset.h>#include <cppconn/statement.h>#include <mysql_connection.h>#include <algorithm>#include <cstddef>#include <cstdint>#include <functional>#include <queue>#include <sstream>#include <string>#include <unordered_map>#include <utility>#include <vector>

Go to the source code of this file.
Classes | |
| class | crawlservpp::Module::Analyzer::Database |
| Class providing database functionality for analyzer threads by implementing Wrapper::Database. More... | |
| class | crawlservpp::Module::Analyzer::Database::Exception |
| Class for analyzer-specific database exceptions. More... | |
Namespaces | |
| crawlservpp::Module::Analyzer | |
| Namespace for analyzer classes. | |
Constants | |
| constexpr auto | crawlservpp::Module::Analyzer::defaultCorpusSlicing {30} |
| The default percentage of the maximum package size allowed by the MySQL server to be used for the maximum size of the corpus. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::corpusSlicingFactor {1.F / 100} |
| The factor used for corpus slicing percentage points (1/100). More... | |
| constexpr auto | crawlservpp::Module::Analyzer::maxNumCorpusColumns {3} |
| The maximum number of columns used when creating a text corpus. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressDeletedCorpus {0.05F} |
| The progress with creating a corpus after the old corpus has been deleted. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressReceivedSources {0.35F} |
| The progress with creating a corpus after the source texts have been received. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressMovedData {0.4F} |
| The progress with creating a corpus after the data has been moved. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressCreatedCorpus {0.6F} |
| The progress with creating a corpus after the server created the corpus. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressSlicedCorpus {0.65F} |
| The progress with creating a corpus after the corpus has been sliced. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressAddingCorpus |
| The remaining progress, attributed to adding the corpus to the database. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressReceivedCorpus {0.8F} |
| The progress with getting an existing corpus after its contents have been received from the database. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressGeneratedSavePoint {0.1F} |
| The progress of saving a savepoint after generating it. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::progressSavingSavePoint |
| The remaining progress, attributed to saving a savepoint to the database. More... | |
Constants for SQL Queries | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg1 {1} |
| First argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg2 {2} |
| Second argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg3 {3} |
| Third argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg4 {4} |
| Fourth argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg5 {5} |
| Fifth argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg6 {6} |
| Sixth argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg7 {7} |
| Seventh argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg8 {8} |
| Eighth argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg9 {9} |
| Ninth argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg10 {10} |
| Tenth argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg11 {11} |
| Eleventh argument in a SQL query. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::sqlArg12 {12} |
| Twelfth argument in a SQL query. More... | |
Constants for Table Columns | |
| constexpr auto | crawlservpp::Module::Analyzer::column1 {0} |
| First column in a table. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::column2 {1} |
| Second column in a table. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::column3 {2} |
| Third column in a table. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::numColumns1 {1} |
| One table column. More... | |
| constexpr auto | crawlservpp::Module::Analyzer::numColumns2 {2} |
| Two table columns. More... | |