crawlserv++
[under development]
Application for crawling and analyzing textual content of websites.
|
Class handling database access for the command-and-control and its threads. More...
#include <Database.hpp>
Classes | |
class | ConnectionException |
Class for database connection exceptions. More... | |
class | Exception |
Class for generic database exceptions. More... | |
class | IncorrectPathException |
Class for incorrect path exceptions. More... | |
class | PrivilegesException |
Class for insufficient privileges exceptions. More... | |
class | StorageEngineException |
Class for storage engine exceptions. More... | |
class | Transaction |
Wrapper class for in-scope transactions. More... | |
class | WrongArgumentsException |
Class for wrong arguments exceptions. More... | |
Friends | |
class | Wrapper::Database |
Allows access to module threads. More... | |
template<class DB > | |
class | Wrapper::DatabaseLock |
Allows access for scoped locking. More... | |
template<class DB > | |
class | Wrapper::DatabaseTryLock |
Allows access for scoped optional locking. More... | |
Construction and Destruction | |
Database (const DatabaseSettings &dbSettings, const std::string &dbModule) | |
Constructor saving settings and setting default values. More... | |
virtual | ~Database () |
Destructor clearing prepared SQL statements and the connection. More... | |
Setters | |
void | setSleepOnError (std::uint64_t seconds) |
Sets the number of seconds to sleep before trying to reconnect after connection loss. More... | |
void | setTimeOut (std::uint64_t milliseconds) |
Sets the maximum execution time for MySQL queries, in milliseconds. More... | |
Getters | |
const DatabaseSettings & | getSettings () const |
Gets the settings of the database. More... | |
const std::string & | getDriverVersion () const |
Gets the MySQL Connector/C++ version string. More... | |
const std::string & | getDataDir () const |
Gets the default data directory. More... | |
std::uint64_t | getMaxAllowedPacketSize () const |
Gets the maximum allowed packet size for communicating with the MySQL server. More... | |
std::uint64_t | getConnectionId () const |
Gets the connection ID. More... | |
Initialization and Update | |
void | connect () |
Establishes a connection to the database and retrieves information about the server and the connection. More... | |
void | initializeSql () |
Runs initializing SQL commands by processing all .sql files in the SQL (sub-)folder. More... | |
void | prepare () |
Prepares SQL statements for getting the last inserted ID, logging and setting the status of a thread. More... | |
void | update () |
Updates the tables with language and version information in the database. More... | |
Logging | |
void | log (const std::string &logEntry) |
Adds a log entry to the database for the current module. More... | |
void | log (const std::string &logModule, const std::string &logEntry) |
Adds a log entry to the database for any module. More... | |
std::uint64_t | getNumberOfLogEntries (const std::string &logModule) |
Gets the number of log entries from the database. More... | |
void | clearLogs (const std::string &logModule) |
Removes log entries from the database. More... | |
Threads | |
std::vector< ThreadDatabaseEntry > | getThreads () |
Gets information about all threads from the database. More... | |
std::uint64_t | addThread (const ThreadOptions &threadOptions) |
Adds information about a new thread to the database and returns its ID. More... | |
std::uint64_t | getThreadRunTime (std::uint64_t threadId) |
Gets the run time of a thread from the database. More... | |
std::uint64_t | getThreadPauseTime (std::uint64_t threadId) |
Gets the pause time of a thread from the database. More... | |
void | setThreadStatus (std::uint64_t threadId, bool threadPaused, const std::string &threadStatusMessage) |
Updates the status of a thread in the database. More... | |
void | setThreadStatus (std::uint64_t threadId, const std::string &threadStatusMessage) |
Updates the status of a thread in the database without changing the pause state of the thread. More... | |
void | setThreadRunTime (std::uint64_t threadId, std::uint64_t threadRunTime) |
Updates the run time of a thread in the database. More... | |
void | setThreadPauseTime (std::uint64_t threadId, std::uint64_t threadPauseTime) |
Updates the pause time of a thread in the database. More... | |
void | deleteThread (std::uint64_t threadId) |
Removes a thread from the database. More... | |
Websites | |
std::uint64_t | addWebsite (const WebsiteProperties &websiteProperties) |
Adds a new website to the database and returns its ID. More... | |
std::string | getWebsiteDomain (std::uint64_t id) |
Gets the domain of a website from the database. More... | |
std::string | getWebsiteNamespace (std::uint64_t websiteId) |
Gets the namespace of a website from the database. More... | |
std::uint64_t | getWebsiteFromUrlList (std::uint64_t listId) |
Gets the ID of the website associated with a URL list from the database. More... | |
IdString | getWebsiteNamespaceFromUrlList (std::uint64_t listId) |
Gets the ID and the namespace of the website associated with a URL list from the database. More... | |
IdString | getWebsiteNamespaceFromConfig (std::uint64_t configId) |
Gets the ID and the namespace of the website associated with a configuration from the database. More... | |
IdString | getWebsiteNamespaceFromTargetTable (const std::string &type, std::uint64_t tableId) |
Gets the ID and the namespace of the website associated with a target table from the database. More... | |
bool | isWebsiteNamespace (const std::string &nameSpace) |
Checks whether a website namespace exists in the database. More... | |
std::string | duplicateWebsiteNamespace (const std::string &websiteNamespace) |
Gets a new name for a website namespace to duplicate. More... | |
std::string | getWebsiteDataDirectory (std::uint64_t websiteId) |
Gets the data directory used by a website. More... | |
std::uint64_t | getChangedUrlsByWebsiteUpdate (std::uint64_t websiteId, const WebsiteProperties &websiteProperties) |
Gets the number of URLs that will be modified by updating a website from the database. More... | |
std::uint64_t | getLostUrlsByWebsiteUpdate (std::uint64_t websiteId, const WebsiteProperties &websiteProperties) |
Gets the number of URLs that will be lost by updating a website from the database. More... | |
void | updateWebsite (std::uint64_t websiteId, const WebsiteProperties &websiteProperties) |
Updates a website and all associated tables in the database. More... | |
void | deleteWebsite (std::uint64_t websiteId) |
Deletes a website and all associated data from the database. More... | |
std::uint64_t | duplicateWebsite (std::uint64_t websiteId, const Queries &queries) |
Duplicates a website, its URL lists, queries, and configurations in the database. More... | |
void | moveWebsite (std::uint64_t websiteId, const WebsiteProperties &websiteProperties) |
Moves a website and all associated data to another data directory in the database. More... | |
URL Lists | |
std::uint64_t | addUrlList (std::uint64_t websiteId, const UrlListProperties &listProperties) |
Adds a new URL list to the database and returns its ID. More... | |
std::queue< IdString > | getUrlLists (std::uint64_t websiteId) |
Gets all URL lists associated with a website from the database. More... | |
std::size_t | mergeUrls (std::uint64_t listId, std::queue< std::string > &urls) |
Inserts URLs that do not already exist into a URL list in the database, returns the number of added URLs. More... | |
std::queue< std::string > | getUrls (std::uint64_t listId) |
Gets all URLs from a URL list in the database. More... | |
std::queue< IdString > | getUrlsWithIds (std::uint64_t listId) |
Gets all URLs and their IDs from a URL list in the database. More... | |
std::string | getUrlListNamespace (std::uint64_t listId) |
Gets the namespace of a URL list from the database. More... | |
IdString | getUrlListNamespaceFromTargetTable (const std::string &type, std::uint64_t tableId) |
Gets the namespace of a URL list from the database by using a target table. More... | |
bool | isUrlListNamespace (std::uint64_t websiteId, const std::string &nameSpace) |
Checks whether a URL list namespace for a specific website exists in the database. More... | |
void | updateUrlList (std::uint64_t listId, const UrlListProperties &listProperties) |
Updates a URL list and all associated tables in the database. More... | |
void | deleteUrlList (std::uint64_t listId) |
Deletes a URL list and all associated data from the database. More... | |
std::size_t | deleteUrls (std::uint64_t listId, std::queue< uint64_t > &urlIds) |
Deletes URLs from a URL list in the database and returns the number of deleted URLs. More... | |
void | resetParsingStatus (std::uint64_t listId) |
Resets the parsing status of all URLs in a URL list in the database. More... | |
void | resetExtractingStatus (std::uint64_t listId) |
Resets the extracting status of all URLs in a URL list in the database. More... | |
void | resetAnalyzingStatus (std::uint64_t listId) |
Resets the analyzing status of all URLs in a URL list in the database. More... | |
Queries | |
std::uint64_t | addQuery (std::uint64_t websiteId, const QueryProperties &queryProperties) |
Adds a new query to the database and returns its ID. More... | |
void | getQueryProperties (std::uint64_t queryId, QueryProperties &queryPropertiesTo) |
Gets the properties of a query from the database. More... | |
void | updateQuery (std::uint64_t queryId, const QueryProperties &queryProperties) |
Updates a query in the database. More... | |
void | moveQuery (std::uint64_t queryId, std::uint64_t toWebsiteId) |
Moves a query to another website in the database. More... | |
void | deleteQuery (std::uint64_t queryId) |
Deletes a query from the database. More... | |
std::uint64_t | duplicateQuery (std::uint64_t queryId) |
Duplicates a query in the database. More... | |
Configurations | |
std::uint64_t | addConfiguration (std::uint64_t websiteId, const ConfigProperties &configProperties) |
Adds a new configuration to the database and returns its ID. More... | |
std::string | getConfiguration (std::uint64_t configId) |
Gets a configuration from the database. More... | |
void | updateConfiguration (std::uint64_t configId, const ConfigProperties &configProperties) |
Updates a configuration in the database. More... | |
void | deleteConfiguration (std::uint64_t configId) |
Deletes a configuration from the database. More... | |
std::uint64_t | duplicateConfiguration (std::uint64_t configId) |
Duplicates a configuration in the database. More... | |
Target Tables | |
std::uint64_t | addOrUpdateTargetTable (const TargetTableProperties &properties) |
Adds a new target table or updates an existing target table in the database. More... | |
std::queue< IdString > | getTargetTables (const std::string &type, std::uint64_t listId) |
Gets the target tables of the specified type for a URL list from the database. More... | |
std::uint64_t | getTargetTableId (const std::string &type, std::uint64_t listId, const std::string &tableName) |
Gets the ID of a target table from the database. More... | |
std::string | getTargetTableName (std::string_view type, std::uint64_t tableId) |
Gets the name of a target table from the database. More... | |
void | deleteTargetTable (const std::string &type, std::uint64_t tableId) |
Deletes a target table from the database. More... | |
Validation | |
void | checkConnection () |
Checks whether the connection to the database is still valid and tries to reconnect if necessary. More... | |
bool | isWebsite (std::uint64_t websiteId) |
Checks whether a website ID is valid. More... | |
bool | isUrlList (std::uint64_t urlListId) |
Checks whether a URL list ID is valid. More... | |
bool | isUrlList (std::uint64_t websiteId, std::uint64_t urlListId) |
Checks whether a URL list ID is valid for a specific website. More... | |
bool | isQuery (std::uint64_t queryId) |
Checks whether a query ID is valid. More... | |
bool | isQuery (std::uint64_t websiteId, std::uint64_t queryId) |
Checks whether a query ID is valid for a specific website. More... | |
bool | isCorpus (std::uint64_t firstId, bool requireArticles, bool requireDates) |
Checks whether a corpus is valid. More... | |
bool | isConfiguration (std::uint64_t configId) |
Checks whether a configuration ID is valid. More... | |
bool | isConfiguration (std::uint64_t websiteId, std::uint64_t configId) |
Checks whether a configuration ID is valid for a specific website. More... | |
bool | isTargetTable (std::string_view type, std::uint64_t websiteId, std::uint64_t urlListId, std::uint64_t tableID) |
Checks whether a target table ID is valid for a specific website and URL list. More... | |
bool | checkDataDir (const std::string &dir) |
Checks whether a data directory is known to the database. More... | |
Locking | |
void | beginNoLock () |
Disables database locking by starting a new SQL transaction. More... | |
void | endNoLock () |
Re-enables database locking by ending the previous SQL transaction. More... | |
Tables | |
bool | isTableEmpty (const std::string &tableName) |
Checks whether a table in the database is empty. More... | |
bool | isTableExists (const std::string &tableName) |
Checks whether a table exists in the database. More... | |
bool | isColumnExists (const std::string &tableName, const std::string &columnName) |
Checks whether a table in the database contains a specific column. More... | |
std::string | getColumnType (const std::string &tableName, const std::string &columnName) |
Gets the type of a specific table column from the database. More... | |
void | readTableAsStrings (const std::string &tableName, std::vector< std::vector< std::string >> &contentsTo, bool includeColumnNames) |
Reads all contents of a table as strings. More... | |
void | readColumnAsStrings (const std::string &tableName, const std::string &columnName, const std::string &condition, std::vector< std::string > &contentsTo) |
Reads all rows of a table column as strings. More... | |
void | lockTables (std::queue< TableNameWriteAccess > &tableLocks) |
Locks tables in the database. More... | |
void | unlockTables () |
Unlocks all previously locked tables in the database. More... | |
void | startTransaction (const std::string &isolationLevel) |
Starts a new transaction with the database using a specific isolation level. More... | |
void | endTransaction (bool success) |
Ends the current transaction with the database, committing the commands on success. More... | |
Custom Data | |
void | getCustomData (Data::GetValue &data) |
Gets a custom value from one column from a table row in the database. More... | |
void | getCustomData (Data::GetFields &data) |
Gets custom values from multiple columns of the same type from a table row. More... | |
void | getCustomData (Data::GetFieldsMixed &data) |
Gets custom values from multiple columns of different types from a table row. More... | |
void | getCustomData (Data::GetColumn &data) |
Gets custom values from a table column in the database. More... | |
void | getCustomData (Data::GetColumns &data) |
Gets custom values from multiple table columns of the same type. More... | |
void | getCustomData (Data::GetColumnsMixed &data) |
Gets custom values from multiple table columns of different types. More... | |
void | insertCustomData (const Data::InsertValue &data) |
Inserts a custom value into a table row in the database. More... | |
void | insertCustomData (const Data::InsertFields &data) |
Inserts custom values into multiple table columns of the same type. More... | |
void | insertCustomData (const Data::InsertFieldsMixed &data) |
Inserts custom values into multiple table columns of different types. More... | |
void | updateCustomData (const Data::UpdateValue &data) |
Updates a custom value in a table row. More... | |
void | updateCustomData (const Data::UpdateFields &data) |
Updates custom values in multiple table columns of the same type. More... | |
void | updateCustomData (const Data::UpdateFieldsMixed &data) |
Updates custom values in multiple table columns of different types. More... | |
Request Counter | |
static std::uint64_t | getRequestCounter () |
Gets the number of SQL requests performed since the start of the application. More... | |
Copy and Move | |
Database (Database &)=delete | |
Deleted copy constructor. More... | |
Database & | operator= (Database &)=delete |
Deleted copy assignment operator. More... | |
Database (Database &&)=delete | |
Deleted move constructor. More... | |
Database & | operator= (Database &&)=delete |
Deleted move assignment operator. More... | |
Shared Connection Information | |
std::unique_ptr< sql::Connection > | connection |
Database connection. More... | |
static sql::Driver * | driver {nullptr} |
Pointer to the MySQL database driver. More... | |
Helper Functions for Prepared SQL Statements | |
void | reserveForPreparedStatements (std::size_t n) |
Reserves memory for a specific number of additional prepared SQL statements. More... | |
void | addPreparedStatement (const std::string &sqlQuery, std::size_t &id) |
Prepares an additional SQL statement and sets its ID. More... | |
void | clearPreparedStatement (std::size_t &id) |
Clears a prepared SQL statement. More... | |
sql::PreparedStatement & | getPreparedStatement (std::size_t id) |
Gets a reference to a prepared SQL statement. More... | |
Database Helper Functions | |
std::uint64_t | getLastInsertedId () |
Gets the last inserted ID from the database. More... | |
void | resetAutoIncrement (const std::string &tableName) |
Resets the auto-increment value of an empty table in the database. More... | |
void | checkDirectory (const std::string &dir) |
Checks access to an external data directory in the database. More... | |
static void | addDatabaseLock (const std::string &name, const IsRunningCallback &isRunningCallback) |
Adds a lock to the database class, blocking execution. More... | |
static bool | tryDatabaseLock (const std::string &name) |
Tries to add a lock to the database class, not blocking execution. More... | |
static void | removeDatabaseLock (const std::string &name) |
Removes a lock from the database class. More... | |
Table Helper Functions | |
void | createTable (const TableProperties &properties) |
Adds a table to the database. More... | |
void | clearTable (const std::string &tableName) |
Clears a table, removing all rows. More... | |
void | dropTable (const std::string &tableName) |
Deletes a table from the database. More... | |
void | addColumn (const std::string &tableName, const TableColumn &column) |
Adds a column to a table in the database. More... | |
void | compressTable (const std::string &tableName) |
Compresses a table in the database. More... | |
std::queue< std::string > | cloneTable (const std::string &tableName, const std::string &destDir) |
Clones a table into another data directory, without copying data or constraints. More... | |
URL List Helper Functions | |
bool | isUrlListCaseSensitive (std::uint64_t listId) |
Gets whether the specified URL list is case-sensitive. More... | |
void | setUrlListCaseSensitive (std::uint64_t listId, bool isCaseSensitive) |
Sets whether the specified URL list is case-sensitive. More... | |
Exception Helper Function | |
static void | sqlException (const std::string &function, const sql::SQLException &e) |
Catches a SQL exception and re-throws it as a specific or a generic Database::Exception. More... | |
Helper Functions for Executing SQL Queries | |
template<class T , class... Args> | |
static bool | sqlExecute (T &statement, Args... args) |
Template function for executing a SQL query. More... | |
template<class T , class... Args> | |
static sql::ResultSet * | sqlExecuteQuery (T &statement, Args... args) |
Template function for executing a SQL query and returning the resulting set. More... | |
template<class T , class... Args> | |
static int | sqlExecuteUpdate (T &statement, Args... args) |
Template function for executing a SQL query and returning the number of affected rows. More... | |
template<class T , class... Args> | |
static bool | sqlExecute (std::unique_ptr< T > &statement, Args... args) |
Template function for executing a SQL query by unique pointer. More... | |
template<class T , class... Args> | |
static sql::ResultSet * | sqlExecuteQuery (std::unique_ptr< T > &statement, Args... args) |
Template function for executing a SQL query by unique pointer and returning the resulting set. More... | |
template<class T , class... Args> | |
static int | sqlExecuteUpdate (std::unique_ptr< T > &statement, Args... args) |
Template function for executing a SQL query by unique pointer and returning the number of affected rows. More... | |
Class handling database access for the command-and-control and its threads.
Thread-specific functionality is not implemented in this (parent) class.
crawlservpp::Main::Database::Database | ( | const DatabaseSettings & | dbSettings, |
const std::string & | dbModule | ||
) |
Constructor saving settings and setting default values.
dbSettings | Constant reference to the database settings. |
dbModule | Constant reference to a string containing the name of the module for logging purposes. |
Main::Database::Exception | if the database instance could not be retrieved. |
References driver.
|
virtual |
Destructor clearing prepared SQL statements and the connection.
Reimplemented in crawlservpp::Module::Database.
References connection, getRequestCounter(), crawlservpp::Helper::CommaLocale::locale(), and log().
|
delete |
Deleted copy constructor.
|
delete |
Deleted move constructor.
|
protected |
Adds a column to a table in the database.
tableName | Constant reference to a string containing the name of the table to which the column will be added. |
column | Constant reference to a structure containing the properties of the column to be added to the table. |
Main::Database::Exception | if no table, column, or column type has been specified, i.e. if one of the strings containing the name of the table, the name of the column, and the type of the column is empty, or if a column reference is incomplete. |
Main::Database::Exception | if a MySQL error occured while adding the column to the given table in the database. |
References checkConnection(), connection, crawlservpp::Struct::TableColumn::name, crawlservpp::Struct::TableColumn::referenceColumn, crawlservpp::Struct::TableColumn::referenceTable, sqlException(), sqlExecute(), and crawlservpp::Struct::TableColumn::type.
Referenced by crawlservpp::Wrapper::Database::addColumn(), and addOrUpdateTargetTable().
std::uint64_t crawlservpp::Main::Database::addConfiguration | ( | std::uint64_t | websiteId, |
const ConfigProperties & | configProperties | ||
) |
Adds a new configuration to the database and returns its ID.
websiteId | The ID of the website to which the new query will be added, or zero for adding a global configuration to the database. |
configProperties | Constant reference to the properties of the new configuration. |
Main::Database::Exception | if no module, name, or configuration has been specfied in the given properties of the new configuration. |
Main::Database::Exception | if a MySQL error occured while adding the new configuration to the database. |
References checkConnection(), crawlservpp::Struct::ConfigProperties::config, connection, getLastInsertedId(), crawlservpp::Struct::ConfigProperties::module, crawlservpp::Struct::ConfigProperties::name, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, crawlservpp::Main::sqlArg4, sqlException(), and sqlExecute().
Referenced by duplicateConfiguration(), duplicateWebsite(), and crawlservpp::Main::Server::tick().
|
staticprotected |
Adds a lock to the database class, blocking execution.
If a lock with the same name already exists, the function will block execution until this lock has been released, or the specified callback function returns false.
name | Constant reference to a string containing the name of the lock to be waited for and added to the database class. |
isRunningCallback | Constant reference to a function that will be regularly called during a block, to enquire whether the thread (or application) is still running. As soon as this function returns false, execution will no longer be blocked, even if the lock could not be added. |
References crawlservpp::Main::sleepOnLockMs.
Referenced by crawlservpp::Wrapper::Database::addDatabaseLock().
std::uint64_t crawlservpp::Main::Database::addOrUpdateTargetTable | ( | const TargetTableProperties & | properties | ) |
Adds a new target table or updates an existing target table in the database.
properties | Constant reference to the properties of the new target table, or the existing target table to be updated. |
Main::Database::Exception | if no type, website, URL list, name, or columns have been specfied in the given properties of the new target table, or if a column of the already existing target table cannot be overwritten due to incompatibilities between the respective data types. |
Main::Database::Exception | if a MySQL error occured while adding the new target table, or updating the existing target table in the database. |
References addColumn(), checkConnection(), crawlservpp::Struct::TargetTableProperties::columns, crawlservpp::Struct::TargetTableProperties::compressed, compressTable(), connection, createTable(), crawlservpp::Struct::TargetTableProperties::fullName, getColumnType(), getLastInsertedId(), getWebsiteDataDirectory(), isColumnExists(), isTableExists(), crawlservpp::Struct::TargetTableProperties::name, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), sqlExecute(), sqlExecuteQuery(), crawlservpp::Struct::TargetTableProperties::type, crawlservpp::Struct::TargetTableProperties::urlList, and crawlservpp::Struct::TargetTableProperties::website.
Referenced by crawlservpp::Wrapper::Database::addOrUpdateTargetTable().
|
protected |
Prepares an additional SQL statement and sets its ID.
If the current ID is not zero, the old prepared statement will be removed.
sqlQuery | Constant reference to a string containing the SQL query for the prepared SQL statement. |
id | Reference to the current ID or zero, which will be set to the new unique ID identifying the prepared SQL query in-class. |
Main::Database::Exception | if a MySQL error occured while preparing and adding the SQL statement. |
std::out_of_range | if id contains an neither zero nor a valid ID. |
References checkConnection(), connection, and sqlException().
Referenced by crawlservpp::Wrapper::Database::addPreparedStatement(), crawlservpp::Module::Database::prepare(), and prepare().
std::uint64_t crawlservpp::Main::Database::addQuery | ( | std::uint64_t | websiteId, |
const QueryProperties & | queryProperties | ||
) |
Adds a new query to the database and returns its ID.
websiteId | The ID of the website to which the new query will be added, or zero for adding a global query to the database. |
queryProperties | Constant reference to the properties of the new query. |
Main::Database::Exception | if no query name, text, or type has been specfied in the given properties of the new query. |
Main::Database::Exception | if a MySQL error occured while adding the new query to the database. |
References checkConnection(), connection, getLastInsertedId(), crawlservpp::Struct::QueryProperties::name, crawlservpp::Struct::QueryProperties::resultBool, crawlservpp::Struct::QueryProperties::resultMulti, crawlservpp::Struct::QueryProperties::resultSingle, crawlservpp::Struct::QueryProperties::resultSubSets, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, crawlservpp::Main::sqlArg4, crawlservpp::Main::sqlArg5, crawlservpp::Main::sqlArg6, crawlservpp::Main::sqlArg7, crawlservpp::Main::sqlArg8, crawlservpp::Main::sqlArg9, sqlException(), sqlExecute(), crawlservpp::Struct::QueryProperties::text, crawlservpp::Struct::QueryProperties::textOnly, and crawlservpp::Struct::QueryProperties::type.
Referenced by duplicateQuery(), duplicateWebsite(), and crawlservpp::Main::Server::tick().
std::uint64_t crawlservpp::Main::Database::addThread | ( | const ThreadOptions & | threadOptions | ) |
Adds information about a new thread to the database and returns its ID.
threadOptions | Constant reference to the options for the new thread. |
Main::Database::Exception | if the module of the thread is empty, or if no module, no website, no URL list, or no configuration has been specified in the given options for the thread. |
Main::Database::Exception | if a MySQL error occured while adding information about the new thread to the database. |
References checkConnection(), crawlservpp::Struct::ThreadOptions::config, connection, getLastInsertedId(), crawlservpp::Struct::ThreadOptions::module, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, crawlservpp::Main::sqlArg4, sqlException(), sqlExecute(), crawlservpp::Struct::ThreadOptions::urlList, and crawlservpp::Struct::ThreadOptions::website.
Referenced by crawlservpp::Module::Thread::Thread().
std::uint64_t crawlservpp::Main::Database::addUrlList | ( | std::uint64_t | websiteId, |
const UrlListProperties & | listProperties | ||
) |
Adds a new URL list to the database and returns its ID.
websiteId | The ID of the website to which the new URL list will be added. |
listProperties | Constant reference to the properties of the new URL list. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero, no namespace or name has been specfied in the given properties of the new URL list, or if the specified URL list namespace already exists in the database. |
Main::Database::Exception | if a MySQL error occured while adding the new URL list to the database. |
References checkConnection(), connection, createTable(), getLastInsertedId(), getWebsiteDataDirectory(), getWebsiteNamespace(), isUrlListNamespace(), crawlservpp::Main::maxColumnsUrlList, crawlservpp::Struct::UrlListProperties::name, crawlservpp::Struct::UrlListProperties::nameSpace, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), and sqlExecute().
Referenced by addWebsite(), duplicateWebsite(), and crawlservpp::Main::Server::tick().
std::uint64_t crawlservpp::Main::Database::addWebsite | ( | const WebsiteProperties & | websiteProperties | ) |
Adds a new website to the database and returns its ID.
websiteProperties | Constant reference to the properties of the new website. |
Main::Database::Exception | if the namespace or the name of the new website is empty, its namespace already exists, or the specified data directory does not exist. |
Main::Database::Exception | if a MySQL error occured while adding the new website to the database. |
References addUrlList(), checkConnection(), connection, deleteWebsite(), crawlservpp::Struct::WebsiteProperties::dir, crawlservpp::Struct::WebsiteProperties::domain, getLastInsertedId(), crawlservpp::Helper::FileSystem::isValidDirectory(), isWebsiteNamespace(), crawlservpp::Struct::WebsiteProperties::name, crawlservpp::Struct::WebsiteProperties::nameSpace, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, crawlservpp::Main::sqlArg4, sqlException(), and sqlExecute().
Referenced by duplicateWebsite(), and crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::beginNoLock | ( | ) |
Disables database locking by starting a new SQL transaction.
Main::Database::Exception | if a MySQL error occured while starting a new SQL transaction in the database. |
References connection, and sqlException().
Referenced by crawlservpp::Wrapper::Database::beginNoLock().
void crawlservpp::Main::Database::checkConnection | ( | ) |
Checks whether the connection to the database is still valid and tries to reconnect if necessary.
Main::Database::Exception | if the MySQL driver is not initialized. |
Main::Database::Exception | if a MySQL error occured while attempting to reconnect to the database. |
References connect(), connection, driver, log(), crawlservpp::Main::reconnectAfterIdleMs, crawlservpp::Helper::DateTime::secondsToString(), crawlservpp::Main::secToMs, sqlException(), and crawlservpp::Timer::Simple::tick().
Referenced by addColumn(), addConfiguration(), addOrUpdateTargetTable(), addPreparedStatement(), addQuery(), addThread(), addUrlList(), addWebsite(), crawlservpp::Wrapper::Database::checkConnection(), checkDirectory(), clearLogs(), clearPreparedStatement(), clearTable(), cloneTable(), compressTable(), createTable(), deleteConfiguration(), deleteQuery(), deleteTargetTable(), deleteThread(), deleteUrlList(), deleteUrls(), deleteWebsite(), dropTable(), duplicateConfiguration(), duplicateQuery(), duplicateWebsite(), endTransaction(), getChangedUrlsByWebsiteUpdate(), getColumnType(), getConfiguration(), getCustomData(), getLastInsertedId(), getLostUrlsByWebsiteUpdate(), getNumberOfLogEntries(), getQueryProperties(), getTargetTableId(), getTargetTableName(), getTargetTables(), getThreadPauseTime(), getThreadRunTime(), getThreads(), getUrlListNamespace(), getUrlListNamespaceFromTargetTable(), getUrlLists(), getUrls(), getUrlsWithIds(), getWebsiteDataDirectory(), getWebsiteDomain(), getWebsiteFromUrlList(), getWebsiteNamespace(), getWebsiteNamespaceFromConfig(), getWebsiteNamespaceFromTargetTable(), insertCustomData(), isColumnExists(), isConfiguration(), isCorpus(), isQuery(), isTableEmpty(), isTableExists(), isTargetTable(), isUrlList(), isUrlListCaseSensitive(), isUrlListNamespace(), isWebsite(), isWebsiteNamespace(), log(), mergeUrls(), moveQuery(), moveWebsite(), crawlservpp::Module::Database::prepare(), readColumnAsStrings(), readTableAsStrings(), resetAnalyzingStatus(), resetAutoIncrement(), resetExtractingStatus(), resetParsingStatus(), crawlservpp::Module::Database::setThreadLast(), setThreadPauseTime(), crawlservpp::Module::Database::setThreadProgress(), setThreadRunTime(), setThreadStatus(), crawlservpp::Module::Database::setThreadStatusMessage(), setTimeOut(), setUrlListCaseSensitive(), sqlException(), startTransaction(), crawlservpp::Main::Server::tick(), update(), updateConfiguration(), updateCustomData(), updateQuery(), updateUrlList(), and updateWebsite().
bool crawlservpp::Main::Database::checkDataDir | ( | const std::string & | dir | ) |
Checks whether a data directory is known to the database.
dir | Constant reference to a string containing the data directory to be checked. |
Referenced by crawlservpp::Main::Server::tick().
|
protected |
Checks access to an external data directory in the database.
Adds and removes a test table to check access to the given data directory.
dir | Constant reference to a string containing the data directory to be checked. |
Main::Database::Exception | if no data directory is specified, i.e. the string containing the directory is empty. |
Main::Database::Exception | if a MySQL error occured while checking access to the data directory in the database. |
References checkConnection(), connection, sqlException(), and sqlExecute().
void crawlservpp::Main::Database::clearLogs | ( | const std::string & | logModule | ) |
Removes log entries from the database.
logModule | Constant reference to a string containing the name of the module, whose log entries will be removed, or to an empty string for all log entries to be removed. |
Main::Database::Exception | if a MySQL error occured while removing log entries from the database. |
References checkConnection(), clearTable(), connection, isTableEmpty(), resetAutoIncrement(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by crawlservpp::Main::Server::tick().
|
protected |
Clears a prepared SQL statement.
id | Reference to the current ID, which will be set to zero after the corresponding prepared SQL statement has been cleared. |
Main::Database::Exception | if a MySQL error occured while clearing the prepared SQL statement. |
std::out_of_range | if id contains zero or an invalid ID. |
References checkConnection(), and sqlException().
Referenced by crawlservpp::Wrapper::Database::clearPreparedStatement().
|
protected |
Clears a table, removing all rows.
tableName | String view containing the name of the table to be cleared. |
Main::Database::Exception | if the table name is invalid, or a MySQL error occured while clearing the table, e.g. if the specified table does not exist. |
References checkConnection(), crawlservpp::Helper::Strings::checkSQLName(), and sqlException().
Referenced by clearLogs(), and dropTable().
|
protected |
Clones a table into another data directory, without copying data or constraints.
<tablename>_tmp
may not exist already in the database!tableName | Const reference to a string containing the name of the table to be cloned. |
destDir | Const reference to a string containing the data directory into which the table will be cloned. |
Main::Database::Exception | if no table is specified, i.e. if the string containing the name of the table is empty, or if the properties of a table could not be retrieved. |
Main::Database::Exception | if a MySQL error occured while cloning the table into the other data directory. |
References checkConnection(), connection, crawlservpp::Main::sqlConstraint, sqlException(), sqlExecute(), sqlExecuteQuery(), and crawlservpp::Helper::Strings::trim().
Referenced by moveWebsite().
|
protected |
Compresses a table in the database.
The function will have no effect om the table, if the table is already compressed.
tableName | Constant reference to a string containing the name of the table to be compressed. |
Main::Database::Exception | if no table is specified, i.e. if the string containing the name of the table is empty, or if a row format could not be determined. |
Main::Database::Exception | if a MySQL error occured while compressing the table in the database. |
References checkConnection(), connection, crawlservpp::Struct::DatabaseSettings::name, sqlException(), sqlExecute(), and sqlExecuteQuery().
Referenced by addOrUpdateTargetTable(), and crawlservpp::Wrapper::Database::compressTable().
void crawlservpp::Main::Database::connect | ( | ) |
Establishes a connection to the database and retrieves information about the server and the connection.
Main::Database::Exception | if the MySQL driver could not be loaded, a connection to the MySQL database could not be established, the connection to the database is invalid, a SQL statement could not be created, a server variable could not be retrieved or has an invalid value, or the connection ID could not be retrieved. |
Main::Database::Exception | if a MySQL error occured while connecting to the MySQL server, or while retrieving information about the server and the connection. |
References crawlservpp::Helper::Container::append(), crawlservpp::Struct::DatabaseSettings::compression, connection, driver, crawlservpp::Helper::FileSystem::getPathSeparator(), crawlservpp::Struct::DatabaseSettings::host, crawlservpp::Main::lockTimeOutSec, crawlservpp::Main::maxContentSize, crawlservpp::Struct::DatabaseSettings::name, crawlservpp::Struct::DatabaseSettings::password, crawlservpp::Struct::DatabaseSettings::port, crawlservpp::Helper::Strings::split(), sqlException(), sqlExecute(), sqlExecuteQuery(), crawlservpp::Helper::Strings::trim(), and crawlservpp::Struct::DatabaseSettings::user.
Referenced by checkConnection(), crawlservpp::Module::Thread::log(), and crawlservpp::Main::Server::tick().
|
protected |
Adds a table to the database.
id
will be created automatically.properties | Constant reference to a structure containing the properties of the table to be created. |
Main::Database::Exception | if no name or columns are specified in the given properties structure, if one of the columns defined there is missing its name or data type, or if a column reference is incomplete. |
Main::Database::Exception | if a MySQL error occured while adding the table to the database. |
References checkConnection(), crawlservpp::Struct::TableProperties::columns, crawlservpp::Struct::TableProperties::compressed, connection, crawlservpp::Struct::TableProperties::dataDirectory, crawlservpp::Struct::TableProperties::name, sqlException(), and sqlExecute().
Referenced by addOrUpdateTargetTable(), addUrlList(), crawlservpp::Wrapper::Database::createTable(), and update().
void crawlservpp::Main::Database::deleteConfiguration | ( | std::uint64_t | configId | ) |
Deletes a configuration from the database.
configId | The ID of the configuration that will be deleted from the database. |
Main::Database::Exception | if no configuration has been specified, i.e. the configuration ID is zero. |
Main::Database::Exception | if a MySQL error occured while deleting the configuration from the database. |
References checkConnection(), connection, isTableEmpty(), resetAutoIncrement(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::deleteQuery | ( | std::uint64_t | queryId | ) |
Deletes a query from the database.
queryId | The ID of the query that will be deleted from the database. |
Main::Database::Exception | if no query has been specified, i.e. the query ID is zero. |
Main::Database::Exception | if a MySQL error occured while deleting the query from the database. |
References checkConnection(), connection, isTableEmpty(), resetAutoIncrement(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::deleteTargetTable | ( | const std::string & | type, |
std::uint64_t | tableId | ||
) |
Deletes a target table from the database.
type | Constant reference to a string containing the type of the target table to be deleted |
tableId | The ID of the target table to be deleted. |
Main::Database::Exception | if no target table has been specified, i.e. the string containing the type is empty or the target table ID is zero. |
Main::Database::Exception | if a MySQL error occured while deleting the target table from the database. |
References checkConnection(), connection, getTargetTableName(), getUrlListNamespaceFromTargetTable(), getWebsiteNamespaceFromTargetTable(), isTableEmpty(), resetAutoIncrement(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by crawlservpp::Wrapper::Database::deleteTargetTable(), and deleteUrlList().
void crawlservpp::Main::Database::deleteThread | ( | std::uint64_t | threadId | ) |
Removes a thread from the database.
threadId | The ID of the thread to be removed from the database. |
Main::Database::Exception | if no thread has been specified, i.e. the thread ID is zero. |
Main::Database::Exception | if a MySQL error occured while deleting the thread from the database. |
References checkConnection(), connection, isTableEmpty(), resetAutoIncrement(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by crawlservpp::Module::Thread::end().
void crawlservpp::Main::Database::deleteUrlList | ( | std::uint64_t | listId | ) |
Deletes a URL list and all associated data from the database.
listId | The ID of the URL list that will be deleted from the database. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while getting the namespace of the URL list, deleting all associated tables, or deleting the URL list from the database. |
References checkConnection(), connection, deleteTargetTable(), dropTable(), getTargetTables(), getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), isTableEmpty(), resetAutoIncrement(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by deleteWebsite(), and crawlservpp::Main::Server::tick().
std::size_t crawlservpp::Main::Database::deleteUrls | ( | std::uint64_t | listId, |
std::queue< uint64_t > & | urlIds | ||
) |
Deletes URLs from a URL list in the database and returns the number of deleted URLs.
Each MySQL query performed by this function will delete a maximum of 100 URLs at once.
listId | The ID of the URL list that will be deleted from the database. |
urlIds | Reference to a queue containing the IDs of the URLs to be deleted. It will be cleared while deleting the URLs, so that the referenced queue will be empty when the deletion succeeds. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while deleting the URLs from the given URL list in the database. |
References checkConnection(), getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), crawlservpp::Main::nAtOnce100, and sqlException().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::deleteWebsite | ( | std::uint64_t | websiteId | ) |
Deletes a website and all associated data from the database.
websiteId | The ID of the website that will be deleted from the database. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero. |
Main::Database::Exception | if a MySQL error occured while getting the namespace of the website, deleting all associated URL lists, or deleting the website from the database. |
References checkConnection(), connection, deleteUrlList(), getUrlLists(), getWebsiteNamespace(), isTableEmpty(), resetAutoIncrement(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by addWebsite(), and crawlservpp::Main::Server::tick().
|
protected |
Deletes a table from the database.
If the table does not exist in the database, the database will not be changed.
tableName | Constant reference to a string containing the name of the table to be deleted, if it exists. |
Main::Database::Exception | if no table has been specified, i.e. if the string containing the name of the table is empty. |
Main::Database::Exception | if a MySQL error occured while removing the table from the database. |
References checkConnection(), clearTable(), connection, isTableExists(), sqlException(), and sqlExecute().
Referenced by deleteUrlList(), crawlservpp::Wrapper::Database::dropTable(), moveWebsite(), and update().
std::uint64_t crawlservpp::Main::Database::duplicateConfiguration | ( | std::uint64_t | configId | ) |
Duplicates a configuration in the database.
configId | The ID of the configuration that will be duplicated in the database. |
Main::Database::Exception | if no configuration has been specified, i.e. the configuration ID is zero. |
Main::Database::Exception | if a MySQL error occured while duplicating the configuration in the database. |
References addConfiguration(), checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
std::uint64_t crawlservpp::Main::Database::duplicateQuery | ( | std::uint64_t | queryId | ) |
Duplicates a query in the database.
queryId | The ID of the query that will be duplicated in the database. |
Main::Database::Exception | if no query has been specified, i.e. the query ID is zero. |
Main::Database::Exception | if a MySQL error occured while duplicating the query in the database. |
References addQuery(), checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
std::uint64_t crawlservpp::Main::Database::duplicateWebsite | ( | std::uint64_t | websiteId, |
const Queries & | queries | ||
) |
Duplicates a website, its URL lists, queries, and configurations in the database.
A namespace for the new website will automatically be created by using duplicateWebsiteNamespace.
A name for the new website will automatically be created by adding " (copy)" to the name of the duplicated website.
websiteId | The ID of the website to duplicate in the database. |
queries | Constant reference to a vector containing pairs of all module names with their queries (i.e. vectors with key, value pairs including the "cat" and "name" keys). This vector will be used to update the queries inside the duplicated configurations. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero, or if an error occured while parsing the duplicated configurations. |
Main::Database::Exception | if a MySQL error occured while getting information about the website, creating the new namespace of the website, or duplicating the website, its URL lists, queries, and configurations. |
References addConfiguration(), addQuery(), addUrlList(), addWebsite(), checkConnection(), connection, duplicateWebsiteNamespace(), crawlservpp::Helper::Json::parseRapid(), crawlservpp::Main::sqlArg1, sqlException(), sqlExecuteQuery(), crawlservpp::Helper::Json::stringify(), and crawlservpp::Main::Exception::view().
Referenced by crawlservpp::Main::Server::tick().
std::string crawlservpp::Main::Database::duplicateWebsiteNamespace | ( | const std::string & | websiteNamespace | ) |
Gets a new name for a website namespace to duplicate.
Adds a number to the end of the namespace. If the namespace already ends on a number, this number will be increased until the new name does not already exist.
websiteNamespace | Const reference to a string containing the name of the website namespace to duplicate. |
Main::Database::Exception | if no namespace has been specified, i.e. the string containing the name of the namespace is empty, or if a conversion from string to number failed. |
Main::Database::Exception | if a MySQL error occured while checking the existence of the website namespace in the database. |
References isWebsiteNamespace().
Referenced by duplicateWebsite().
void crawlservpp::Main::Database::endNoLock | ( | ) |
Re-enables database locking by ending the previous SQL transaction.
Main::Database::Exception | if a MySQL error occured while ending the previous SQL transaction by committing the changes to the database. |
References connection, and sqlException().
Referenced by crawlservpp::Wrapper::Database::endNoLock().
void crawlservpp::Main::Database::endTransaction | ( | bool | success | ) |
Ends the current transaction with the database, committing the commands on success.
Rolls back the commands if the transaction was not successful.
success | Specifies whether the transaction was successful and the commands should be committed. |
Main::Database::Exception | if a MySQL error occured while starting the SQL transaction. |
References checkConnection(), connection, and sqlException().
std::uint64_t crawlservpp::Main::Database::getChangedUrlsByWebsiteUpdate | ( | std::uint64_t | websiteId, |
const WebsiteProperties & | websiteProperties | ||
) |
Gets the number of URLs that will be modified by updating a website from the database.
websiteId | The ID of the website for which the number of URLs that will be modified will be retrieved from the database. |
websiteProperties | A constant reference to the new website properties. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero, or if no website namespace or name has been specified in the provided website options. |
Main::Database::Exception | if a MySQL error occured while retrieving the number of URLs that will be modified from the database. |
References checkConnection(), connection, crawlservpp::Struct::WebsiteProperties::domain, getUrlLists(), getWebsiteDomain(), getWebsiteNamespace(), crawlservpp::Struct::WebsiteProperties::name, crawlservpp::Struct::WebsiteProperties::nameSpace, sqlException(), sqlExecuteQuery(), and crawlservpp::Main::wwwPrefix.
Referenced by crawlservpp::Main::Server::tick().
std::string crawlservpp::Main::Database::getColumnType | ( | const std::string & | tableName, |
const std::string & | columnName | ||
) |
Gets the type of a specific table column from the database.
tableName | Constant reference to a string containing the name of the table in the database from which the type of the column will be retrieved. |
columnName | Constant reference to a string containing the name of the column whose type will be retrieved. |
Main::Database::Exception | if no table or column has been specified, i.e. one of the strings containing the name and the column is empty. |
Main::Database::Exception | if a MySQL error occured while retrieving the type of the given column, e.g. if the specified table does not exist. |
References checkConnection(), connection, crawlservpp::Struct::DatabaseSettings::name, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), and sqlExecuteQuery().
Referenced by addOrUpdateTargetTable(), and crawlservpp::Wrapper::Database::getColumnType().
std::string crawlservpp::Main::Database::getConfiguration | ( | std::uint64_t | configId | ) |
Gets a configuration from the database.
configId | The ID of the configuration to be retrieved from the database. |
Main::Database::Exception | if no configuration has been specified, i.e. the configuration ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the configuration from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Wrapper::Database::getConfiguration(), crawlservpp::Module::Thread::Thread(), and crawlservpp::Main::Server::tick().
std::uint64_t crawlservpp::Main::Database::getConnectionId | ( | ) | const |
Gets the connection ID.
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::getCustomData | ( | Data::GetValue & | data | ) |
Gets a custom value from one column from a table row in the database.
data | Reference to the data structure that identifies the column, and to which the result will be written. |
Main::Database::Exception | if no column name or no column type is specified in the given data structure, or if an invalid data type has been encountered. |
Main::Database::Exception | if a MySQL error occured while retrieving the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::GetValue::column, crawlservpp::Data::GetValue::condition, connection, sqlException(), sqlExecuteQuery(), crawlservpp::Data::GetValue::table, crawlservpp::Data::GetValue::type, and crawlservpp::Data::GetValue::value.
Referenced by crawlservpp::Wrapper::Database::getCustomData(), and crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::getCustomData | ( | Data::GetFields & | data | ) |
Gets custom values from multiple columns of the same type from a table row.
data | Reference to the data structure that identifies the columns, and to which the result will be written. |
Main::Database::Exception | if no column names or no column type are specified in the given data structure, or if an invalid data type has been encountered. |
Main::Database::Exception | if a MySQL error occured while retrieving the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::GetFields::columns, crawlservpp::Data::GetFields::condition, connection, sqlException(), sqlExecuteQuery(), crawlservpp::Data::GetFields::table, crawlservpp::Data::GetFields::type, and crawlservpp::Data::GetFields::values.
void crawlservpp::Main::Database::getCustomData | ( | Data::GetFieldsMixed & | data | ) |
Gets custom values from multiple columns of different types from a table row.
data | Reference to the data structure that identifies the columns and their types, and to which the result will be written. |
Main::Database::Exception | if no columns are specified in the given data structure, or if an invalid data type has been encountered. |
Main::Database::Exception | if a MySQL error occured while retrieving the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, checkConnection(), crawlservpp::Data::GetFieldsMixed::columns_types, crawlservpp::Data::GetFieldsMixed::condition, connection, sqlException(), sqlExecuteQuery(), crawlservpp::Data::GetFieldsMixed::table, and crawlservpp::Data::GetFieldsMixed::values.
void crawlservpp::Main::Database::getCustomData | ( | Data::GetColumn & | data | ) |
Gets custom values from a table column in the database.
data | Reference to the data structure that identifies the column, and to which the result will be written. |
Main::Database::Exception | if no column or column type is specified in the given data structure, or if an invalid data type has been encountered. |
Main::Database::Exception | if a MySQL error occured while retrieving the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::GetColumn::column, crawlservpp::Data::GetColumn::condition, connection, crawlservpp::Data::GetColumn::order, sqlException(), sqlExecuteQuery(), crawlservpp::Data::GetColumn::table, crawlservpp::Data::GetColumn::type, and crawlservpp::Data::GetColumn::values.
void crawlservpp::Main::Database::getCustomData | ( | Data::GetColumns & | data | ) |
Gets custom values from multiple table columns of the same type.
data | Reference to the data structure that identifies the columns, and to which the result will be written. |
Main::Database::Exception | if no column or column type is specified in the given data structure, or if an invalid data type has been encountered. |
Main::Database::Exception | if a MySQL error occured while retrieving the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::GetColumns::columns, crawlservpp::Data::GetColumns::condition, connection, crawlservpp::Data::GetColumns::order, sqlException(), sqlExecuteQuery(), crawlservpp::Data::GetColumns::table, crawlservpp::Data::GetColumns::type, and crawlservpp::Data::GetColumns::values.
void crawlservpp::Main::Database::getCustomData | ( | Data::GetColumnsMixed & | data | ) |
Gets custom values from multiple table columns of different types.
data | Reference to the data structure that identifies the columns and their types, and to which the result will be written. |
Main::Database::Exception | if no columns have been specified in the given data structure, or if an invalid data type has been encountered. |
Main::Database::Exception | if a MySQL error occured while retrieving the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, checkConnection(), crawlservpp::Data::GetColumnsMixed::columns_types, crawlservpp::Data::GetColumnsMixed::condition, connection, crawlservpp::Data::GetColumnsMixed::order, sqlException(), sqlExecuteQuery(), crawlservpp::Data::GetColumnsMixed::table, and crawlservpp::Data::GetColumnsMixed::values.
const std::string & crawlservpp::Main::Database::getDataDir | ( | ) | const |
Gets the default data directory.
Referenced by crawlservpp::Main::Server::tick().
const std::string & crawlservpp::Main::Database::getDriverVersion | ( | ) | const |
Gets the MySQL Connector/C++ version string.
Referenced by crawlservpp::Main::Server::tick().
|
protected |
Gets the last inserted ID from the database.
Main::Database::Exception | if the prepared SQL statement for retrieving the last inserted ID from the database is missing. |
Main::Database::Exception | if a MySQL error occured while retrieving the last inserted ID from the database. |
References checkConnection(), getPreparedStatement(), sqlException(), and sqlExecuteQuery().
Referenced by addConfiguration(), addOrUpdateTargetTable(), addQuery(), addThread(), addUrlList(), addWebsite(), and crawlservpp::Wrapper::Database::getLastInsertedId().
std::uint64_t crawlservpp::Main::Database::getLostUrlsByWebsiteUpdate | ( | std::uint64_t | websiteId, |
const WebsiteProperties & | websiteProperties | ||
) |
Gets the number of URLs that will be lost by updating a website from the database.
websiteId | The ID of the website for which the number of URLs that will be lost will be retrieved from the database. |
websiteProperties | A constant reference to the new website properties. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero, or if no website namespace or name has been specified in the provided website options. |
Main::Database::Exception | if a MySQL error occured while retrieving the number of URLs that will be lost from the database. |
References checkConnection(), connection, crawlservpp::Struct::WebsiteProperties::domain, getUrlLists(), getWebsiteDomain(), getWebsiteNamespace(), crawlservpp::Struct::WebsiteProperties::name, crawlservpp::Struct::WebsiteProperties::nameSpace, sqlException(), sqlExecuteQuery(), and crawlservpp::Main::wwwPrefix.
Referenced by crawlservpp::Main::Server::tick().
std::uint64_t crawlservpp::Main::Database::getMaxAllowedPacketSize | ( | ) | const |
Gets the maximum allowed packet size for communicating with the MySQL server.
Referenced by crawlservpp::Wrapper::Database::getMaxAllowedPacketSize(), insertCustomData(), crawlservpp::Main::Server::tick(), and updateCustomData().
std::uint64_t crawlservpp::Main::Database::getNumberOfLogEntries | ( | const std::string & | logModule | ) |
Gets the number of log entries from the database.
The number can be retrieved for a specific module or for all modules at once.
logModule | Constant reference to a string containing the name of the module, for which the number of log entries will be retrieved, or to an empty string for all log entries to be retrieved. |
Main::Database::Exception | if a MySQL error occured while getting the number of log entries from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
|
protected |
Gets a reference to a prepared SQL statement.
id | The ID of the prepared SQL statement to retrieve. |
Main::Database::Exception | if a MySQL error occured while retrieving the prepared SQL statement. |
References sqlException().
Referenced by getLastInsertedId(), crawlservpp::Wrapper::Database::getPreparedStatement(), log(), crawlservpp::Module::Database::setThreadLast(), crawlservpp::Module::Database::setThreadProgress(), setThreadStatus(), and crawlservpp::Module::Database::setThreadStatusMessage().
void crawlservpp::Main::Database::getQueryProperties | ( | std::uint64_t | queryId, |
QueryProperties & | queryPropertiesTo | ||
) |
Gets the properties of a query from the database.
queryId | The ID of the query for which the properties will be retrieved from the database. |
queryPropertiesTo | Reference to the structure to which the retrieved properties of the query will be written. |
Main::Database::Exception | if no query ID has been specfied, i.e. the query ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the properties of the given query from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Wrapper::Database::getQueryProperties(), and crawlservpp::Main::Server::tick().
|
inlinestatic |
Gets the number of SQL requests performed since the start of the application.
Referenced by crawlservpp::Wrapper::Database::getRequestCounter(), and ~Database().
const Database::DatabaseSettings & crawlservpp::Main::Database::getSettings | ( | ) | const |
Gets the settings of the database.
std::uint64_t crawlservpp::Main::Database::getTargetTableId | ( | const std::string & | type, |
std::uint64_t | listId, | ||
const std::string & | tableName | ||
) |
Gets the ID of a target table from the database.
type | Constant reference to a string containing the type of the target table for which to retrieve its ID. |
listId | The ID of the URL list associated with the target table for which to retrieve its ID. |
tableName | Const reference to a string containing the name of the target table for which to retrieve its ID. |
Main::Database::Exception | if no target table or URL list has been specified, i.e. if the string containing the type is empty, or the target table or the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the ID of the target table from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Wrapper::Database::getTargetTableId().
std::string crawlservpp::Main::Database::getTargetTableName | ( | std::string_view | type, |
std::uint64_t | tableId | ||
) |
Gets the name of a target table from the database.
type | String view containing the type of the target table for which to retrieve its name. |
tableId | The ID of the target table for which to retrieve its name. |
Main::Database::Exception | if no target table has been specified, i.e. the string containing the type is empty or the target table ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the name of the target table from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by deleteTargetTable(), crawlservpp::Wrapper::Database::getTargetTableName(), and crawlservpp::Main::Server::tick().
std::queue< Database::IdString > crawlservpp::Main::Database::getTargetTables | ( | const std::string & | type, |
std::uint64_t | listId | ||
) |
Gets the target tables of the specified type for a URL list from the database.
type | Constant reference to a string containing the type of the target tables to retrieve. |
listId | The ID of the URL list for which to retrieve the target tables. |
Main::Database::Exception | if no target table has been specified, i.e. the string containing the type is empty or the target table ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the target tables from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by deleteUrlList(), crawlservpp::Wrapper::Database::getTargetTables(), moveWebsite(), updateUrlList(), and updateWebsite().
std::uint64_t crawlservpp::Main::Database::getThreadPauseTime | ( | std::uint64_t | threadId | ) |
Gets the pause time of a thread from the database.
threadId | The ID of the thread for which the pause time will be retrieved from the database. |
Main::Database::Exception | if no thread has been specified, i.e. the thread ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the pause time of the thread from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Module::Thread::log().
std::uint64_t crawlservpp::Main::Database::getThreadRunTime | ( | std::uint64_t | threadId | ) |
Gets the run time of a thread from the database.
threadId | The ID of the thread for which the run time will be retrieved from the database. |
Main::Database::Exception | if no thread has been specified, i.e. the thread ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the run time of the thread from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Module::Thread::log().
std::vector< Database::ThreadDatabaseEntry > crawlservpp::Main::Database::getThreads | ( | ) |
Gets information about all threads from the database.
Main::Database::Exception | if a MySQL error occured while getting information about the threads from the database. |
References checkConnection(), connection, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
std::string crawlservpp::Main::Database::getUrlListNamespace | ( | std::uint64_t | listId | ) |
Gets the namespace of a URL list from the database.
listId | The ID of the URL list whose namespace will be retrieved. |
Main::Database::Exception | if no URL list has been specified, i.e. the list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the namespace of the URL list. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by deleteUrlList(), deleteUrls(), getUrlListNamespaceFromTargetTable(), getUrls(), getUrlsWithIds(), mergeUrls(), resetAnalyzingStatus(), resetExtractingStatus(), resetParsingStatus(), crawlservpp::Module::Thread::Thread(), crawlservpp::Main::Server::tick(), and updateUrlList().
Database::IdString crawlservpp::Main::Database::getUrlListNamespaceFromTargetTable | ( | const std::string & | type, |
std::uint64_t | tableId | ||
) |
Gets the namespace of a URL list from the database by using a target table.
type | Constant reference to a string containing the type of the target table whose URL list's namespace will be retrieved. |
tableId | The ID of the target table whose URL list's namespace will be retrieved. |
Main::Database::Exception | if no target table type has been specified, i.e. the type is empty, or no target table has been specified, i.e. the string containing the type is empty or the target table ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the namespace of the URL list. |
References checkConnection(), connection, getUrlListNamespace(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by deleteTargetTable().
std::queue< Database::IdString > crawlservpp::Main::Database::getUrlLists | ( | std::uint64_t | websiteId | ) |
Gets all URL lists associated with a website from the database.
websiteId | The ID of the website for which the associated URL lists will be retrieved. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the URL lists from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by deleteWebsite(), getChangedUrlsByWebsiteUpdate(), getLostUrlsByWebsiteUpdate(), moveWebsite(), and updateWebsite().
std::queue< std::string > crawlservpp::Main::Database::getUrls | ( | std::uint64_t | listId | ) |
Gets all URLs from a URL list in the database.
listId | The ID of the URL list from which the URLs will be retrieved. |
Main::Database::Exception | if no URL list has been specified, i.e. the list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the URLs from the given URL list in the database. |
References checkConnection(), connection, getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
std::queue< Database::IdString > crawlservpp::Main::Database::getUrlsWithIds | ( | std::uint64_t | listId | ) |
Gets all URLs and their IDs from a URL list in the database.
listId | The ID of the URL list from which the URLs and their IDs will be retrieved. |
Main::Database::Exception | if no URL list has been specified, i.e. the list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the URLs and their IDs from the given URL list in the database. |
References checkConnection(), connection, getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
std::string crawlservpp::Main::Database::getWebsiteDataDirectory | ( | std::uint64_t | websiteId | ) |
Gets the data directory used by a website.
websiteId | The ID of the website for which the data directory will be retrieved from the database. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the data directory of the website from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by addOrUpdateTargetTable(), addUrlList(), crawlservpp::Main::Server::tick(), and updateWebsite().
std::string crawlservpp::Main::Database::getWebsiteDomain | ( | std::uint64_t | websiteId | ) |
Gets the domain of a website from the database.
websiteId | The ID of the website for which the domain will be retrieved from the database. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the domain name of the given website from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by getChangedUrlsByWebsiteUpdate(), getLostUrlsByWebsiteUpdate(), crawlservpp::Wrapper::Database::getWebsiteDomain(), crawlservpp::Main::Server::tick(), and updateWebsite().
std::uint64_t crawlservpp::Main::Database::getWebsiteFromUrlList | ( | std::uint64_t | listId | ) |
Gets the ID of the website associated with a URL list from the database.
listId | The ID of the URL list for which the ID of the associated website will be retrieved from the database. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the ID of the website associated with the given URL list from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by getWebsiteNamespaceFromUrlList(), and crawlservpp::Main::Server::tick().
std::string crawlservpp::Main::Database::getWebsiteNamespace | ( | std::uint64_t | websiteId | ) |
Gets the namespace of a website from the database.
websiteId | The ID of the website for which the namespace will be retrieved from the database. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the namespace of the given website from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by addUrlList(), deleteWebsite(), getChangedUrlsByWebsiteUpdate(), getLostUrlsByWebsiteUpdate(), getWebsiteNamespaceFromConfig(), getWebsiteNamespaceFromTargetTable(), getWebsiteNamespaceFromUrlList(), crawlservpp::Module::Thread::Thread(), crawlservpp::Main::Server::tick(), and updateWebsite().
Database::IdString crawlservpp::Main::Database::getWebsiteNamespaceFromConfig | ( | std::uint64_t | configId | ) |
Gets the ID and the namespace of the website associated with a configuration from the database.
configId | The ID of the configuration for which the ID and namespace of the associated website will be retrieved from the database. |
Main::Database::Exception | if no configuration has been specified, i.e. the configuration ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the ID and namespace of the website associated with the given configuration from the database. |
References checkConnection(), connection, getWebsiteNamespace(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Database::IdString crawlservpp::Main::Database::getWebsiteNamespaceFromTargetTable | ( | const std::string & | type, |
std::uint64_t | tableId | ||
) |
Gets the ID and the namespace of the website associated with a target table from the database.
type | Constant reference to a string containing the type of the target table for which the ID and namespace of the associated website will be retrieved from the database. |
tableId | The ID of the target table for which the ID and namespace of the associated website will be retrieved from the database. |
Main::Database::Exception | if no target table has been specified, i.e. the string containing the type is empty or the target table ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the ID and namespace of the website associated with the given target table from the database. |
References checkConnection(), connection, getWebsiteNamespace(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by deleteTargetTable().
Database::IdString crawlservpp::Main::Database::getWebsiteNamespaceFromUrlList | ( | std::uint64_t | listId | ) |
Gets the ID and the namespace of the website associated with a URL list from the database.
listId | The ID of the URL list for which the ID and namespace of the associated website will be retrieved from the database. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the ID and namespace of the website associated with the given URL list from the database. |
References getWebsiteFromUrlList(), and getWebsiteNamespace().
Referenced by deleteUrlList(), deleteUrls(), getUrls(), getUrlsWithIds(), mergeUrls(), resetAnalyzingStatus(), resetExtractingStatus(), resetParsingStatus(), and updateUrlList().
void crawlservpp::Main::Database::initializeSql | ( | ) |
Runs initializing SQL commands by processing all .sql files in the SQL (sub-)folder.
Main::Database::Exception | if no SQL statement could be created or a .sql file could not be opened for reading. |
Main::Database::Exception | if a MySQL error occured while executing the commands inside the .sql files. |
References crawlservpp::Helper::FileSystem::listFilesInPath(), crawlservpp::Main::sqlDir, and crawlservpp::Main::sqlExtension.
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::insertCustomData | ( | const Data::InsertValue & | data | ) |
Inserts a custom value into a table row in the database.
data | Constant reference to a structure containing the data to be inserted. |
Main::Database::Exception | if no table, columns, or column type have been specified in the given data structure, if the given data is too large, or if an invalid data has been encountered. |
Main::Database::Exception | if a MySQL error occured while inserting the data. |
References crawlservpp::Data::Value::_b, crawlservpp::Data::_bool, crawlservpp::Data::Value::_d, crawlservpp::Data::_double, crawlservpp::Data::Value::_i32, crawlservpp::Data::Value::_i64, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::Value::_isnull, crawlservpp::Data::Value::_s, crawlservpp::Data::_string, crawlservpp::Data::Value::_ui32, crawlservpp::Data::Value::_ui64, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::InsertValue::column, connection, getMaxAllowedPacketSize(), crawlservpp::Helper::CommaLocale::locale(), crawlservpp::Main::maxContentSize, crawlservpp::Main::maxContentSizeString, crawlservpp::Main::sqlArg1, sqlException(), sqlExecute(), crawlservpp::Data::InsertValue::table, crawlservpp::Data::InsertValue::type, and crawlservpp::Data::InsertValue::value.
Referenced by crawlservpp::Wrapper::Database::insertCustomData().
void crawlservpp::Main::Database::insertCustomData | ( | const Data::InsertFields & | data | ) |
Inserts custom values into multiple table columns of the same type.
data | Constant reference to a structure containing the data to be inserted. |
Main::Database::Exception | if no table, columns, or column type have been specified in the given data structure, if the given data is too large, or if an invalid data has been encountered. |
Main::Database::Exception | if a MySQL error occured while inserting the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::InsertFields::columns_values, connection, getMaxAllowedPacketSize(), crawlservpp::Helper::CommaLocale::locale(), crawlservpp::Main::maxContentSize, crawlservpp::Main::maxContentSizeString, crawlservpp::Main::sqlArg1, sqlException(), sqlExecute(), crawlservpp::Data::InsertFields::table, and crawlservpp::Data::InsertFields::type.
void crawlservpp::Main::Database::insertCustomData | ( | const Data::InsertFieldsMixed & | data | ) |
Inserts custom values into multiple table columns of different types.
data | Constant reference to a structure containing the data to be inserted. |
Main::Database::Exception | if no table or columns have been specified in the given data structure, if the given data is too large, or if an invalid data has been encountered. |
Main::Database::Exception | if a MySQL error occured while inserting the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, checkConnection(), crawlservpp::Data::InsertFieldsMixed::columns_types_values, connection, getMaxAllowedPacketSize(), crawlservpp::Helper::CommaLocale::locale(), crawlservpp::Main::maxContentSize, crawlservpp::Main::maxContentSizeString, sqlException(), sqlExecute(), and crawlservpp::Data::InsertFieldsMixed::table.
bool crawlservpp::Main::Database::isColumnExists | ( | const std::string & | tableName, |
const std::string & | columnName | ||
) |
Checks whether a table in the database contains a specific column.
tableName | Constant reference to a string containing the name of the table in the database in which the existence of the column will be checked. |
columnName | Constant reference to a string containing the name of the column to be checked for in the given table. |
Main::Database::Exception | if no table or columns have been specified, i.e. one of the strings containing the name and the column is empty. |
Main::Database::Exception | if a MySQL error occured while checking the existence of the given column, e.g. if the specified table does not exist. |
References checkConnection(), connection, crawlservpp::Struct::DatabaseSettings::name, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), and sqlExecuteQuery().
Referenced by addOrUpdateTargetTable(), crawlservpp::Module::Analyzer::Database::checkSources(), and crawlservpp::Wrapper::Database::isColumnExists().
bool crawlservpp::Main::Database::isConfiguration | ( | std::uint64_t | configId | ) |
Checks whether a configuration ID is valid.
configId | The configuration ID to be checked for in the database. |
Main::Database::Exception | if no configuration has been specified, i.e. the configuration ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given configuration ID in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
bool crawlservpp::Main::Database::isConfiguration | ( | std::uint64_t | websiteId, |
std::uint64_t | configId | ||
) |
Checks whether a configuration ID is valid for a specific website.
websiteId | The ID of the website to which the configuration should belong. |
configId | The configuration ID to be checked for in the database. |
Main::Database::Exception | if no website or no configuration has been specified, i.e. the website ID or the configuration ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given configuration ID in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecuteQuery().
bool crawlservpp::Main::Database::isCorpus | ( | std::uint64_t | firstId, |
bool | requireArticles, | ||
bool | requireDates | ||
) |
Checks whether a corpus is valid.
firstId | The ID of the first chunk belonging to the corpus. |
requireArticle | The corpus needs to include an article map. |
requireDates | The corpus needs to include a date map. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
bool crawlservpp::Main::Database::isQuery | ( | std::uint64_t | queryId | ) |
Checks whether a query ID is valid.
queryId | The query ID to be checked for in the database. |
Main::Database::Exception | if no query has been specified, i.e. the query ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given query ID in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
bool crawlservpp::Main::Database::isQuery | ( | std::uint64_t | websiteId, |
std::uint64_t | queryId | ||
) |
Checks whether a query ID is valid for a specific website.
websiteId | The ID of the website to which the query should belong. |
queryId | The query ID to be checked for in the database. |
Main::Database::Exception | if no website or no query has been specified, i.e. the website ID or the query ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given query ID in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecuteQuery().
bool crawlservpp::Main::Database::isTableEmpty | ( | const std::string & | tableName | ) |
Checks whether a table in the database is empty.
tableName | Constant reference to a string containing the name of the table whose contents will be checked in the database. |
Main::Database::Exception | if no table has been specified, i.e. the string containing the name is empty. |
Main::Database::Exception | if a MySQL error occured while checking the content of the given table in the database, e.g. if the table does not exist. |
References checkConnection(), connection, sqlException(), and sqlExecuteQuery().
Referenced by clearLogs(), deleteConfiguration(), deleteQuery(), deleteTargetTable(), deleteThread(), deleteUrlList(), deleteWebsite(), and crawlservpp::Wrapper::Database::isTableEmpty().
bool crawlservpp::Main::Database::isTableExists | ( | const std::string & | tableName | ) |
Checks whether a table exists in the database.
tableName | Constant reference to a string containing the name of the table whose existence in the database will be checked. |
Main::Database::Exception | if no table has been specified, i.e. the string containing the name is empty. |
Main::Database::Exception | if a MySQL error occured while checking the existence of the given table in the database. |
References checkConnection(), connection, crawlservpp::Struct::DatabaseSettings::name, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecuteQuery().
Referenced by addOrUpdateTargetTable(), crawlservpp::Module::Analyzer::Database::checkSources(), dropTable(), and crawlservpp::Wrapper::Database::isTableExists().
bool crawlservpp::Main::Database::isTargetTable | ( | std::string_view | type, |
std::uint64_t | websiteId, | ||
std::uint64_t | urlListId, | ||
std::uint64_t | tableId | ||
) |
Checks whether a target table ID is valid for a specific website and URL list.
type | String view containing the type of the target table to be checked. |
websiteId | The ID of the website to which the target table should belong. |
urlListId | The ID of the URL list to which the target table should belong. |
tableId | The ID to be checked for in the database. |
Main::Database::Exception | if no table type, website, URL list, or target table has been specified, i.e. the website ID, URL list ID, or the target table ID is zero, or the target table type is empty or contains invalid characters. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given target table ID in the database. |
References checkConnection(), crawlservpp::Helper::Strings::checkSQLName(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
bool crawlservpp::Main::Database::isUrlList | ( | std::uint64_t | urlListId | ) |
Checks whether a URL list ID is valid.
urlListId | The URL list ID to be checked for in the database. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given URL list ID in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
bool crawlservpp::Main::Database::isUrlList | ( | std::uint64_t | websiteId, |
std::uint64_t | urlListId | ||
) |
Checks whether a URL list ID is valid for a specific website.
websiteId | The ID of the website to which the URL list should belong. |
urlListId | The URL list ID to be checked for in the database. |
Main::Database::Exception | if no website or no URL list has been specified, i.e. the website ID or the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given URL list ID in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecuteQuery().
|
protected |
Gets whether the specified URL list is case-sensitive.
listId | The ID of the URL list to be checked for case-sensitivity. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the case-sensitivity of the URL list from the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by mergeUrls().
bool crawlservpp::Main::Database::isUrlListNamespace | ( | std::uint64_t | websiteId, |
const std::string & | nameSpace | ||
) |
Checks whether a URL list namespace for a specific website exists in the database.
websiteId | The ID of the website for which the URL list namespace will be checked. |
nameSpace | Constant reference to a string containing the URL list namespace to be checked. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero, or no URL list namespace has been specified, i.e. the namespace is empty. |
Main::Database::Exception | if a MySQL error occured while checking the existence of the URL list namespace in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecuteQuery().
Referenced by addUrlList(), and updateUrlList().
bool crawlservpp::Main::Database::isWebsite | ( | std::uint64_t | websiteId | ) |
Checks whether a website ID is valid.
websiteId | The website ID to be checked for in the database. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the validity of the given website ID in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
bool crawlservpp::Main::Database::isWebsiteNamespace | ( | const std::string & | nameSpace | ) |
Checks whether a website namespace exists in the database.
nameSpace | Constant reference to a string containing the namespace whose existence will be checked in the database. |
Main::Database::Exception | if no namespace has been specified, i.e. the string containing the name of the namespace is empty. |
Main::Database::Exception | if a MySQL error occured while checking the existence of the website namespace in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, sqlException(), and sqlExecuteQuery().
Referenced by addWebsite(), duplicateWebsiteNamespace(), and updateWebsite().
void crawlservpp::Main::Database::lockTables | ( | std::queue< TableNameWriteAccess > & | tableLocks | ) |
Locks tables in the database.
tableLocks | Reference to a queue of string-bool pairs containing the table names and whether to grant write access to each table. |
Main::Database::Exception | if a MySQL error occured while locking the given tables, e.g. if another table lock is active. |
References connection, and sqlException().
void crawlservpp::Main::Database::log | ( | const std::string & | logEntry | ) |
Adds a log entry to the database for the current module.
Removes invalid UTF-8 characters if necessary.
logEntry | Constant reference to a string containing the log entry to be added to the database. |
Main::Database::Exception | if a MySQL error occured while adding the log entry to the database. If such an exception is thrown, the log entry will be written to stdout instead. |
Referenced by checkConnection(), crawlservpp::Module::Database::log(), crawlservpp::Main::Server::tick(), and ~Database().
void crawlservpp::Main::Database::log | ( | const std::string & | logModule, |
const std::string & | logEntry | ||
) |
Adds a log entry to the database for any module.
Removes invalid UTF-8 characters if necessary. If characters needed to be removed, a note will be included in the log entry that will be added to the database.
logModule | Constant reference to a string containing the name of the module from which the log entry is added to the database. |
logEntry | Constant reference to a string containing the log entry to be added to the database. |
Main::Database::Exception | if the prepared SQL statement for adding a log entry is missing. |
Main::Database::Exception | if a MySQL error occured while adding the log entry to the database. If such an exception is thrown, the log entry will be written to stdout instead. |
References checkConnection(), getPreparedStatement(), crawlservpp::Helper::Utf8::repairUtf8(), crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecute().
std::size_t crawlservpp::Main::Database::mergeUrls | ( | std::uint64_t | listId, |
std::queue< std::string > & | urls | ||
) |
Inserts URLs that do not already exist into a URL list in the database, returns the number of added URLs.
listId | The ID of the URL list to which those URLs will be added that the list does not already contain. |
urls | Reference to a queue containing the URLs to be inserted, if they do not already exist in the given URL list. The queue will be emptied, even if URLs will not be added, because they already exist in the URL list. |
Main::Database::Exception | if no URL list has been specified, i.e. the list ID is zero. |
Main::Database::Exception | if a MySQL error occured while checking the existence of the URLs and adding the non-existing URLs to the URL list. |
References checkConnection(), connection, getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), isUrlListCaseSensitive(), crawlservpp::Main::nAtOnce500, crawlservpp::Main::numArgsAddUrl, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, crawlservpp::Main::sqlArg4, sqlException(), and sqlExecuteUpdate().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::moveQuery | ( | std::uint64_t | queryId, |
std::uint64_t | toWebsiteId | ||
) |
Moves a query to another website in the database.
queryId | The ID of the query that will be moved to another website in the database. |
toWebsiteId | The ID of the website to which the query will be moved in the database, or zero to change the query into a global query. |
Main::Database::Exception | if no query has been specified, i.e. the query ID is zero. |
Main::Database::Exception | if a MySQL error occured while moving the query in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecute().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::moveWebsite | ( | std::uint64_t | websiteId, |
const WebsiteProperties & | websiteProperties | ||
) |
Moves a website and all associated data to another data directory in the database.
Creates temporary copies of all tables first, enough memory needs therefore to be available if both data directories are located on the same physical device.
websiteId | The ID of the website that will be moved to another data directory in the database. |
websiteProperties | Constant reference to the properties of the website to be moved, including the new data directory. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero. |
Main::Database::Exception | if a MySQL error occured while moving the website and all associated data to another data directory in the database. |
References checkConnection(), cloneTable(), connection, crawlservpp::Struct::WebsiteProperties::dir, dropTable(), getTargetTables(), getUrlLists(), crawlservpp::Struct::WebsiteProperties::name, crawlservpp::Struct::DatabaseSettings::name, crawlservpp::Struct::WebsiteProperties::nameSpace, crawlservpp::Main::nAtOnce10, crawlservpp::Main::nAtOnce100, crawlservpp::Main::numUrlListTables, sqlException(), sqlExecute(), sqlExecuteQuery(), crawlservpp::Main::Database::Transaction::success(), crawlservpp::Timer::Simple::tickStr(), and crawlservpp::Main::Exception::view().
Referenced by updateWebsite().
Deleted copy assignment operator.
Deleted move assignment operator.
void crawlservpp::Main::Database::prepare | ( | ) |
Prepares SQL statements for getting the last inserted ID, logging and setting the status of a thread.
Main::Database::Exception | if a MySQL error occured while preparing those SQL statements. |
References addPreparedStatement(), reserveForPreparedStatements(), and sqlException().
Referenced by crawlservpp::Module::Database::prepare(), and crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::readColumnAsStrings | ( | const std::string & | tableName, |
const std::string & | columnName, | ||
const std::string & | condition, | ||
std::vector< std::string > & | contentsTo | ||
) |
Reads all rows of a table column as strings.
tableName | Constant reference to a string containing the name of the table in the database from which the row contents will be retrieved. |
columnName | Constant reference to a string containing the name of the table column in the database from which the contents will be retrieved. |
condition | Constant reference to a string containing the SQL condition for a row to be returned. If it references an empty string, all rows will be returned. |
contentsTo | A reference to a vector of strings to which to append the values of all retrieved cells. |
Main::Database::Exception | if the table name or column is invalid, or a MySQL error occured while reading the column, e.g. if the specified table or column does not exist. |
References checkConnection(), crawlservpp::Helper::Strings::checkSQLName(), connection, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::readTableAsStrings | ( | const std::string & | tableName, |
std::vector< std::vector< std::string >> & | contentsTo, | ||
bool | includeColumnNames | ||
) |
Reads all contents of a table as strings.
tableName | Constant reference to a string containing the name of the table in the database from which the contents will be retrieved. |
contentsTo | A reference to a vector of rows represented by vectors of strings to which to append the values of all retrieved cells. |
includeColumnNames | Specifies whether the column names will be included in the resulting table. |
Main::Database::Exception | if the table name is invalid, or a MySQL error occured while reading the table, e.g. if the specified table does not exist. |
References checkConnection(), crawlservpp::Helper::Strings::checkSQLName(), connection, sqlException(), and sqlExecuteQuery().
Referenced by crawlservpp::Main::Server::tick().
|
staticprotected |
Removes a lock from the database class.
Does nothing if a lock with the given name does not exist in the database class.
name | Constant reference to a string containing the name of the lock to be removed from the database class. |
Referenced by crawlservpp::Wrapper::Database::removeDatabaseLock().
|
protected |
Reserves memory for a specific number of additional prepared SQL statements.
n | Number of prepared SQL statements for which memory should be reserved. |
Referenced by crawlservpp::Module::Database::prepare(), prepare(), and crawlservpp::Wrapper::Database::reserveForPreparedStatements().
void crawlservpp::Main::Database::resetAnalyzingStatus | ( | std::uint64_t | listId | ) |
Resets the analyzing status of all URLs in a URL list in the database.
listId | The ID of the URL list for which the analyzing status will be reset. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while resetting the analyzing state of all URLs in the given URL list. |
References checkConnection(), getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), and sqlException().
Referenced by crawlservpp::Main::Server::tick().
|
protected |
Resets the auto-increment value of an empty table in the database.
tableName | Const reference to a string containing the name of the table whose auto-increment value will be reset. |
Main::Database::Exception | if no table is specified, i.e. the string containing the name of the table is empty. * |
Main::Database::Exception | if a MySQL error occured while resetting the auto-increment value of the table in the database. |
References checkConnection(), connection, sqlException(), and sqlExecute().
Referenced by clearLogs(), deleteConfiguration(), deleteQuery(), deleteTargetTable(), deleteThread(), deleteUrlList(), and deleteWebsite().
void crawlservpp::Main::Database::resetExtractingStatus | ( | std::uint64_t | listId | ) |
Resets the extracting status of all URLs in a URL list in the database.
listId | The ID of the URL list for which the extracting status will be reset. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while resetting the extracting state of all URLs in the given URL list. |
References checkConnection(), getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), and sqlException().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::resetParsingStatus | ( | std::uint64_t | listId | ) |
Resets the parsing status of all URLs in a URL list in the database.
listId | The ID of the URL list for which the parsing status will be reset. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while resetting the parsing state of all URLs in the given URL list. |
References checkConnection(), getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), and sqlException().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::setSleepOnError | ( | std::uint64_t | seconds | ) |
Sets the number of seconds to sleep before trying to reconnect after connection loss.
seconds | The number of seconds to wait before trying to reconnect to the MySQL server after the connection got lost. |
Referenced by crawlservpp::Wrapper::Database::setSleepOnError(), and crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::setThreadPauseTime | ( | std::uint64_t | threadId, |
std::uint64_t | threadPauseTime | ||
) |
Updates the pause time of a thread in the database.
threadId | The ID of the thread for which the pause time will be updated in the database. |
threadPauseTime | The total number of seconds the given thread has been paused. |
Main::Database::Exception | if no thread has been specified, i.e. the thread ID is zero. |
Main::Database::Exception | if a MySQL error occured while updating the pause time of the thread in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecute().
Referenced by crawlservpp::Module::Thread::log().
void crawlservpp::Main::Database::setThreadRunTime | ( | std::uint64_t | threadId, |
std::uint64_t | threadRunTime | ||
) |
Updates the run time of a thread in the database.
threadId | The ID of the thread for which the run time will be updated in the database. |
threadRunTime | The total number of seconds the given thread has been running, and not been paused. |
Main::Database::Exception | if no thread has been specified, i.e. the thread ID is zero. |
Main::Database::Exception | if a MySQL error occured while updating the run time of the thread in the database. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecute().
Referenced by crawlservpp::Module::Thread::log().
void crawlservpp::Main::Database::setThreadStatus | ( | std::uint64_t | threadId, |
bool | threadPaused, | ||
const std::string & | threadStatusMessage | ||
) |
Updates the status of a thread in the database.
Adds the pause state to the status message if necessary.
threadId | The ID of the thread for which the status will be updated in the database. |
threadPaused | Specifies whether the thread is currently paused. |
threadStatusMessage | Constant reference to a string containing the current status of the thread. |
Main::Database::Exception | if no thread has been specified, i.e. the thread ID is zero, or the prepared SQL statement for setting the thread status is missing. |
Main::Database::Exception | if a MySQL error occured while updating the status of the thread in the database. |
References checkConnection(), getPreparedStatement(), crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), and sqlExecute().
Referenced by crawlservpp::Module::Thread::pause(), crawlservpp::Module::Thread::pauseByThread(), crawlservpp::Module::Thread::setStatusMessage(), crawlservpp::Module::Thread::Thread(), and crawlservpp::Module::Thread::unpause().
void crawlservpp::Main::Database::setThreadStatus | ( | std::uint64_t | threadId, |
const std::string & | threadStatusMessage | ||
) |
Updates the status of a thread in the database without changing the pause state of the thread.
threadId | The ID of the thread for which the status will be updated in the database. |
threadStatusMessage | Constant reference to a string containing the current status of the thread. |
Main::Database::Exception | if no thread has been specified, i.e. the thread ID is zero, or the prepared SQL statement for setting the thread status is missing. |
Main::Database::Exception | if a MySQL error occured while updating the status of the thread in the database. |
References checkConnection(), getPreparedStatement(), crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecute().
void crawlservpp::Main::Database::setTimeOut | ( | std::uint64_t | milliseconds | ) |
Sets the maximum execution time for MySQL queries, in milliseconds.
milliseconds | The number of milliseconds for a MySQL query to run before it gets cancelled, or zero to disable the time-out for MySQL queries. |
Main::Database::Exception | if a MySQL error occurs while setting the execution time. |
References checkConnection(), connection, sqlException(), and sqlExecute().
Referenced by crawlservpp::Wrapper::Database::setTimeOut().
|
protected |
Sets whether the specified URL list is case-sensitive.
listId | The ID of the URL list whose case-sensitivity will be changed. |
isCaseSensitive | Specify whether URLs in the given URL list will be case-sensitive or not. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while setting the case-sensitivity of the URL list. |
References checkConnection(), connection, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, sqlException(), and sqlExecute().
Referenced by crawlservpp::Wrapper::Database::setUrlListCaseSensitive().
|
staticprotected |
Catches a SQL exception and re-throws it as a specific or a generic Database::Exception.
function | Constant reference to a string containing the name of the function in which the exception has been thrown. |
e | Constant reference to the SQL exception that has been thrown. |
References checkConnection(), connection, crawlservpp::Main::sqlAbortedConnection, crawlservpp::Main::sqlBadHandShake, crawlservpp::Main::sqlCannotConnectToServer, crawlservpp::Main::sqlCannotConnectToServerThroughSocket, crawlservpp::Main::sqlCannotCreateIPSocket, crawlservpp::Main::sqlCannotGetHostName, crawlservpp::Main::sqlClientErrorConnectingToMaster, crawlservpp::Main::sqlClientErrorConnectingToSlave, crawlservpp::Main::sqlCouldNotUncompressPackets, crawlservpp::Main::sqlErrorInServerHandshake, crawlservpp::Main::sqlErrorReadingPackets, crawlservpp::Main::sqlErrorWritingPackets, sqlExecute(), sqlExecuteUpdate(), crawlservpp::Main::sqlForcingCloseOfThread, crawlservpp::Main::sqlGotSignal, crawlservpp::Main::sqlIncorrectPath, crawlservpp::Main::sqlInsufficientPrivileges, crawlservpp::Main::sqlInvalidConnectionHandle, crawlservpp::Main::sqlLockWaitTimeOutExceeded, crawlservpp::Main::sqlLostConnectionDuringQuery, crawlservpp::Main::sqlMalformedPacket, crawlservpp::Main::sqlMoreThanMaxUserConnections, crawlservpp::Main::sqlNetErrorReadingFromMaster, crawlservpp::Main::sqlNetErrorWritingToMaster, crawlservpp::Main::sqlNewAbortedConnection, crawlservpp::Main::sqlNormalShutdown, crawlservpp::Main::sqlNumOfLocksExceedsLockTableSize, crawlservpp::Main::sqlPacketsOutOfOrder, crawlservpp::Main::sqlQueryExecutionInterrupted, crawlservpp::Main::sqlReadErrorFromConnectionPipe, crawlservpp::Main::sqlServerErrorConnectingToMaster, crawlservpp::Main::sqlServerHasGoneAway, crawlservpp::Main::sqlServerShutDown, crawlservpp::Main::sqlShutDownComplete, crawlservpp::Main::sqlSortAborted, crawlservpp::Main::sqlSSLConnectionError, crawlservpp::Main::sqlStorageEngineError, crawlservpp::Main::sqlTCPError, crawlservpp::Main::sqlTimeOutReadingPackets, crawlservpp::Main::sqlTimeOutWritingPackets, crawlservpp::Main::sqlTooManyConnections, crawlservpp::Main::sqlUnableToConnectToForeignDataSource, crawlservpp::Main::sqlUnknownServerHost, and crawlservpp::Main::sqlWrongArguments.
Referenced by addColumn(), addConfiguration(), addOrUpdateTargetTable(), addPreparedStatement(), addQuery(), addThread(), addUrlList(), addWebsite(), beginNoLock(), checkConnection(), checkDirectory(), clearLogs(), clearPreparedStatement(), clearTable(), cloneTable(), compressTable(), connect(), createTable(), deleteConfiguration(), deleteQuery(), deleteTargetTable(), deleteThread(), deleteUrlList(), deleteUrls(), deleteWebsite(), dropTable(), duplicateConfiguration(), duplicateQuery(), duplicateWebsite(), endNoLock(), endTransaction(), getChangedUrlsByWebsiteUpdate(), getColumnType(), getConfiguration(), getCustomData(), getLastInsertedId(), getLostUrlsByWebsiteUpdate(), getNumberOfLogEntries(), getPreparedStatement(), getQueryProperties(), getTargetTableId(), getTargetTableName(), getTargetTables(), getThreadPauseTime(), getThreadRunTime(), getThreads(), getUrlListNamespace(), getUrlListNamespaceFromTargetTable(), getUrlLists(), getUrls(), getUrlsWithIds(), getWebsiteDataDirectory(), getWebsiteDomain(), getWebsiteFromUrlList(), getWebsiteNamespace(), getWebsiteNamespaceFromConfig(), getWebsiteNamespaceFromTargetTable(), insertCustomData(), isColumnExists(), isConfiguration(), isCorpus(), isQuery(), isTableEmpty(), isTableExists(), isTargetTable(), isUrlList(), isUrlListCaseSensitive(), isUrlListNamespace(), isWebsite(), isWebsiteNamespace(), lockTables(), log(), mergeUrls(), moveQuery(), moveWebsite(), prepare(), readColumnAsStrings(), readTableAsStrings(), resetAnalyzingStatus(), resetAutoIncrement(), resetExtractingStatus(), resetParsingStatus(), crawlservpp::Module::Database::setThreadLast(), setThreadPauseTime(), crawlservpp::Module::Database::setThreadProgress(), setThreadRunTime(), setThreadStatus(), crawlservpp::Module::Database::setThreadStatusMessage(), setTimeOut(), setUrlListCaseSensitive(), crawlservpp::Wrapper::Database::sqlException(), startTransaction(), unlockTables(), update(), updateConfiguration(), updateCustomData(), updateQuery(), updateUrlList(), and updateWebsite().
|
inlinestaticprotected |
Template function for executing a SQL query.
statement | Reference to the SQL statement. Can be either a prepared statement or a sql::Statement . |
args | Optional arguments to the underlying function executing the query. When a sql::Statement is used, this should be a null- terminated string containing the text of the SQL query. |
References crawlservpp::Wrapper::Database::getRequestCounter(), crawlservpp::Main::sleepOnDeadLockMs, and crawlservpp::Main::sqlDeadLock.
Referenced by addColumn(), addConfiguration(), addOrUpdateTargetTable(), addQuery(), addThread(), addUrlList(), addWebsite(), checkDirectory(), clearLogs(), cloneTable(), compressTable(), connect(), createTable(), deleteConfiguration(), deleteQuery(), deleteTargetTable(), deleteThread(), deleteUrlList(), deleteWebsite(), dropTable(), insertCustomData(), log(), moveQuery(), moveWebsite(), resetAutoIncrement(), crawlservpp::Module::Database::setThreadLast(), setThreadPauseTime(), crawlservpp::Module::Database::setThreadProgress(), setThreadRunTime(), setThreadStatus(), crawlservpp::Module::Database::setThreadStatusMessage(), setTimeOut(), setUrlListCaseSensitive(), sqlException(), crawlservpp::Wrapper::Database::sqlExecute(), update(), updateConfiguration(), updateCustomData(), updateQuery(), updateUrlList(), and updateWebsite().
|
inlinestaticprotected |
Template function for executing a SQL query by unique pointer.
statement | Reference to the unique pointer. to the SQL statement. |
args | Optional arguments to the underlying function executing the query. When a sql::Statement is used, this should be a null- terminated string containing the text of the SQL query. |
|
inlinestaticprotected |
Template function for executing a SQL query and returning the resulting set.
statement | Reference to the SQL statement. Can be either a prepared statement or a sql::Statement . |
args | Optional arguments to the underlying function executing the query. When a sql::Statement is used, this should be a null- terminated string containing the text of the SQL query. |
References crawlservpp::Wrapper::Database::getRequestCounter(), crawlservpp::Main::sleepOnDeadLockMs, and crawlservpp::Main::sqlDeadLock.
Referenced by addOrUpdateTargetTable(), cloneTable(), compressTable(), connect(), duplicateConfiguration(), duplicateQuery(), duplicateWebsite(), getChangedUrlsByWebsiteUpdate(), getColumnType(), getConfiguration(), getCustomData(), getLastInsertedId(), getLostUrlsByWebsiteUpdate(), getNumberOfLogEntries(), getQueryProperties(), getTargetTableId(), getTargetTableName(), getTargetTables(), getThreadPauseTime(), getThreadRunTime(), getThreads(), getUrlListNamespace(), getUrlListNamespaceFromTargetTable(), getUrlLists(), getUrls(), getUrlsWithIds(), getWebsiteDataDirectory(), getWebsiteDomain(), getWebsiteFromUrlList(), getWebsiteNamespace(), getWebsiteNamespaceFromConfig(), getWebsiteNamespaceFromTargetTable(), isColumnExists(), isConfiguration(), isCorpus(), isQuery(), isTableEmpty(), isTableExists(), isTargetTable(), isUrlList(), isUrlListCaseSensitive(), isUrlListNamespace(), isWebsite(), isWebsiteNamespace(), moveWebsite(), readColumnAsStrings(), readTableAsStrings(), and crawlservpp::Wrapper::Database::sqlExecuteQuery().
|
inlinestaticprotected |
Template function for executing a SQL query by unique pointer and returning the resulting set.
statement | Reference to the unique pointer. to the SQL statement. |
args | Optional arguments to the underlying function executing the query. When a sql::Statement is used, this should be a null- terminated string containing the text of the SQL query. |
|
inlinestaticprotected |
Template function for executing a SQL query and returning the number of affected rows.
statement | Reference to the SQL statement. Can be either a prepared statement or a sql::Statement . |
args | Optional arguments to the underlying function executing the query. When a sql::Statement is used, this should be a null- terminated string containing the text of the query. |
References crawlservpp::Wrapper::Database::getRequestCounter(), crawlservpp::Main::sleepOnDeadLockMs, and crawlservpp::Main::sqlDeadLock.
Referenced by mergeUrls(), sqlException(), and crawlservpp::Wrapper::Database::sqlExecuteUpdate().
|
inlinestaticprotected |
Template function for executing a SQL query by unique pointer and returning the number of affected rows.
statement | Reference to the unique pointer. to the SQL statement. |
args | Optional arguments to the underlying function executing the query. When a sql::Statement is used, this should be a null- terminated string containing the text of the SQL query. |
void crawlservpp::Main::Database::startTransaction | ( | const std::string & | isolationLevel | ) |
Starts a new transaction with the database using a specific isolation level.
isolationLevel | Constant reference to a string containing the isolation level to be used, or to an empty string if the default isolation level should be used. |
Main::Database::Exception | if a MySQL error occured while starting the SQL transaction. |
References checkConnection(), connection, and sqlException().
|
staticprotected |
Tries to add a lock to the database class, not blocking execution.
If a lock with the same name already exists, the function will not add a lock and return false instead.
name | Constant reference to a string containing the name of the lock to be added to the database class if a lock with the same name does not exist already. |
Referenced by crawlservpp::Wrapper::Database::tryDatabaseLock().
void crawlservpp::Main::Database::unlockTables | ( | ) |
Unlocks all previously locked tables in the database.
Main::Database::Exception | if a MySQL error occured while unlocking all previously locked tables. |
References connection, and sqlException().
void crawlservpp::Main::Database::update | ( | ) |
Updates the tables with language and version information in the database.
Creates the tables if they do not exist yet.
Main::Database::Exception | if a MySQL error occured while creating or updating those tables. |
References checkConnection(), connection, createTable(), dropTable(), crawlservpp::Helper::Portability::enumLocales(), crawlservpp::Helper::Versions::getLibraryVersions(), crawlservpp::Main::Version::getString(), crawlservpp::Main::sqlArg1, sqlException(), and sqlExecute().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::updateConfiguration | ( | std::uint64_t | configId, |
const ConfigProperties & | configProperties | ||
) |
Updates a configuration in the database.
configId | The ID of the configuration to be updated in the database. |
configProperties | Constant reference to the new properties of the configuration. The name of the module specified in these properties will be ignored. |
Main::Database::Exception | if no configuration has been specified, i.e. the configuration ID is zero, or if no name or configuration has been specified in the given properties of the configuration. |
Main::Database::Exception | if a MySQL error occured while updating the configuration in the database. |
References checkConnection(), crawlservpp::Struct::ConfigProperties::config, connection, crawlservpp::Struct::ConfigProperties::name, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), and sqlExecute().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::updateCustomData | ( | const Data::UpdateValue & | data | ) |
Updates a custom value in a table row.
data | Constant reference to a structure containing the data to be updated. |
Main::Database::Exception | if no table, columns, or column type have been specified in the given data structure, if the given data is too large, or if an invalid data has been encountered. |
Main::Database::Exception | if a MySQL error occured while updating the data. |
References crawlservpp::Data::Value::_b, crawlservpp::Data::_bool, crawlservpp::Data::Value::_d, crawlservpp::Data::_double, crawlservpp::Data::Value::_i32, crawlservpp::Data::Value::_i64, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::Value::_isnull, crawlservpp::Data::Value::_s, crawlservpp::Data::_string, crawlservpp::Data::Value::_ui32, crawlservpp::Data::Value::_ui64, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::UpdateValue::column, crawlservpp::Data::UpdateValue::condition, connection, getMaxAllowedPacketSize(), crawlservpp::Helper::CommaLocale::locale(), crawlservpp::Main::maxContentSize, crawlservpp::Main::maxContentSizeString, crawlservpp::Main::sqlArg1, sqlException(), sqlExecute(), crawlservpp::Data::UpdateValue::table, crawlservpp::Data::UpdateValue::type, and crawlservpp::Data::UpdateValue::value.
Referenced by crawlservpp::Wrapper::Database::updateCustomData().
void crawlservpp::Main::Database::updateCustomData | ( | const Data::UpdateFields & | data | ) |
Updates custom values in multiple table columns of the same type.
data | Constant reference to a structure containing the data to be updated. |
Main::Database::Exception | if no columns or no column type are specified in the given data structure, if the given data is too large, or if an invalid data has been encountered. |
Main::Database::Exception | if a MySQL error occured while updating the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, crawlservpp::Data::_unknown, checkConnection(), crawlservpp::Data::UpdateFields::columns_values, crawlservpp::Data::UpdateFields::condition, connection, getMaxAllowedPacketSize(), crawlservpp::Helper::CommaLocale::locale(), crawlservpp::Main::maxContentSize, crawlservpp::Main::maxContentSizeString, crawlservpp::Main::sqlArg1, sqlException(), sqlExecute(), crawlservpp::Data::UpdateFields::table, and crawlservpp::Data::UpdateFields::type.
void crawlservpp::Main::Database::updateCustomData | ( | const Data::UpdateFieldsMixed & | data | ) |
Updates custom values in multiple table columns of different types.
data | Constant reference to a structure containing the data to be updated. |
Main::Database::Exception | if no columns are specified in the given data structure, if the given data is too large, or if an invalid data has been encountered. |
Main::Database::Exception | if a MySQL error occured while updating the data. |
References crawlservpp::Data::_bool, crawlservpp::Data::_double, crawlservpp::Data::_int32, crawlservpp::Data::_int64, crawlservpp::Data::_string, crawlservpp::Data::_uint32, crawlservpp::Data::_uint64, checkConnection(), crawlservpp::Data::UpdateFieldsMixed::columns_types_values, crawlservpp::Data::UpdateFieldsMixed::condition, connection, getMaxAllowedPacketSize(), crawlservpp::Helper::CommaLocale::locale(), crawlservpp::Main::maxContentSize, crawlservpp::Main::maxContentSizeString, crawlservpp::Main::sqlArg1, sqlException(), sqlExecute(), and crawlservpp::Data::UpdateFieldsMixed::table.
void crawlservpp::Main::Database::updateQuery | ( | std::uint64_t | queryId, |
const QueryProperties & | queryProperties | ||
) |
Updates a query in the database.
Replaces the current query properties with the given ones.
queryId | The ID of the query that will be updated in the database. |
queryProperties | A constant reference to the new query properties. |
Main::Database::Exception | if no query has been specified, i.e. the query ID is zero, or if no name, namespace or text has been specified in the provided query properties. |
Main::Database::Exception | if a MySQL error occured while updating the query in the database. |
References checkConnection(), connection, crawlservpp::Struct::QueryProperties::name, crawlservpp::Struct::QueryProperties::resultBool, crawlservpp::Struct::QueryProperties::resultMulti, crawlservpp::Struct::QueryProperties::resultSingle, crawlservpp::Struct::QueryProperties::resultSubSets, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, crawlservpp::Main::sqlArg4, crawlservpp::Main::sqlArg5, crawlservpp::Main::sqlArg6, crawlservpp::Main::sqlArg7, crawlservpp::Main::sqlArg8, crawlservpp::Main::sqlArg9, sqlException(), sqlExecute(), crawlservpp::Struct::QueryProperties::text, crawlservpp::Struct::QueryProperties::textOnly, and crawlservpp::Struct::QueryProperties::type.
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::updateUrlList | ( | std::uint64_t | listId, |
const UrlListProperties & | listProperties | ||
) |
Updates a URL list and all associated tables in the database.
Replaces the current URL list properties with the given ones.
Changing the namespace will result in the renaming of all associated tables.
listId | The ID of the URL list that will be updated in the database. |
listProperties | A constant reference to the new URL list properties. |
Main::Database::Exception | if no URL list has been specified, i.e. the URL list ID is zero, if no URL list namespace or name has been specified in the provided website options, or if the new namespace already exists in the database. |
Main::Database::Exception | if a MySQL error occured while updating the URL list and all associated tables in the database. |
References checkConnection(), connection, getTargetTables(), getUrlListNamespace(), getWebsiteNamespaceFromUrlList(), isUrlListNamespace(), crawlservpp::Struct::UrlListProperties::name, crawlservpp::Struct::UrlListProperties::nameSpace, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, sqlException(), and sqlExecute().
Referenced by crawlservpp::Main::Server::tick().
void crawlservpp::Main::Database::updateWebsite | ( | std::uint64_t | websiteId, |
const WebsiteProperties & | websiteProperties | ||
) |
Updates a website and all associated tables in the database.
Replaces the current website properties with the given ones.
Changing the namespace will result in the renaming of all associated tables.
URLs will be changed, if the type of the domain changes from using a specific domain to cross-domain, or vice versa, because the name of the domain is saved as part of the URLs if the website is cross-domain.
websiteId | The ID of the website that will be updated in the database. |
websiteProperties | A constant reference to the new website properties. |
Main::Database::Exception | if no website has been specified, i.e. the website ID is zero, if no website namespace or name has been specified in the provided website options, or if the new namespace already exists in the database. |
Main::Database::Exception | if a MySQL error occured while updating the website, its URLs and all associated tables in the database. |
References checkConnection(), connection, crawlservpp::Struct::WebsiteProperties::dir, crawlservpp::Struct::WebsiteProperties::domain, getTargetTables(), getUrlLists(), getWebsiteDataDirectory(), getWebsiteDomain(), getWebsiteNamespace(), isWebsiteNamespace(), moveWebsite(), crawlservpp::Struct::WebsiteProperties::name, crawlservpp::Struct::WebsiteProperties::nameSpace, crawlservpp::Main::sqlArg1, crawlservpp::Main::sqlArg2, crawlservpp::Main::sqlArg3, crawlservpp::Main::sqlArg4, sqlException(), sqlExecute(), and crawlservpp::Main::wwwPrefix.
Referenced by crawlservpp::Main::Server::tick().
|
friend |
Allows access to module threads.
|
friend |
Allows access for scoped locking.
|
friend |
Allows access for scoped optional locking.
|
protected |
Database connection.
Referenced by addColumn(), addConfiguration(), addOrUpdateTargetTable(), addPreparedStatement(), addQuery(), addThread(), addUrlList(), addWebsite(), beginNoLock(), checkConnection(), checkDirectory(), clearLogs(), cloneTable(), compressTable(), connect(), createTable(), deleteConfiguration(), deleteQuery(), deleteTargetTable(), deleteThread(), deleteUrlList(), deleteWebsite(), dropTable(), duplicateConfiguration(), duplicateQuery(), duplicateWebsite(), endNoLock(), endTransaction(), getChangedUrlsByWebsiteUpdate(), getColumnType(), getConfiguration(), getCustomData(), getLostUrlsByWebsiteUpdate(), getNumberOfLogEntries(), getQueryProperties(), getTargetTableId(), getTargetTableName(), getTargetTables(), getThreadPauseTime(), getThreadRunTime(), getThreads(), getUrlListNamespace(), getUrlListNamespaceFromTargetTable(), getUrlLists(), getUrls(), getUrlsWithIds(), getWebsiteDataDirectory(), getWebsiteDomain(), getWebsiteFromUrlList(), getWebsiteNamespace(), getWebsiteNamespaceFromConfig(), getWebsiteNamespaceFromTargetTable(), insertCustomData(), isColumnExists(), isConfiguration(), isCorpus(), isQuery(), isTableEmpty(), isTableExists(), isTargetTable(), isUrlList(), isUrlListCaseSensitive(), isUrlListNamespace(), isWebsite(), isWebsiteNamespace(), lockTables(), mergeUrls(), moveQuery(), moveWebsite(), readColumnAsStrings(), readTableAsStrings(), resetAutoIncrement(), setThreadPauseTime(), setThreadRunTime(), setTimeOut(), setUrlListCaseSensitive(), sqlException(), startTransaction(), unlockTables(), update(), updateConfiguration(), updateCustomData(), updateQuery(), updateUrlList(), updateWebsite(), and ~Database().
|
staticprotected |
Pointer to the MySQL database driver.
Referenced by checkConnection(), connect(), crawlservpp::Module::Database::Database(), Database(), and crawlservpp::Module::Database::~Database().