Wrapper class providing the database functionality of Module::Database to its child classes.
More...
#include <Database.hpp>
|
void | setLogging (std::uint8_t level, std::uint8_t min, std::uint8_t verbose) |
| Sets the current, minimal, and verbose logging levels. More...
|
|
void | setSleepOnError (std::uint64_t seconds) |
| Sets the number of seconds to sleep before trying to reconnect after connection loss. More...
|
|
void | setTimeOut (std::uint64_t milliseconds) |
| Sets the maximum execution time for MySQL queries, in milliseconds. More...
|
|
|
void | log (std::uint8_t level, const std::string &logEntry) |
| Writes a thread-specific log entry to the database. More...
|
|
void | log (std::uint8_t level, std::queue< std::string > &logEntries) |
| Writes multiple thread-specific log entries to the database. More...
|
|
|
std::uint64_t | addOrUpdateTargetTable (const TargetTableProperties &properties) |
| Adds a new target table or updates an existing target table in the database. More...
|
|
std::queue< IdString > | getTargetTables (const std::string &type, std::uint64_t listId) |
| Gets the target tables of the specified type for a URL list from the database. More...
|
|
std::uint64_t | getTargetTableId (const std::string &type, std::uint64_t listId, const std::string &tableName) |
| Gets the ID of a target table from the database. More...
|
|
std::string | getTargetTableName (const std::string &type, std::uint64_t tableId) |
| Gets the name of a target table from the database. More...
|
|
void | addTargetColumn (const std::string &tableName, const TableColumn &column) |
| Adds a column to the target table, if it does not exist already. More...
|
|
void | deleteTargetTable (const std::string &type, std::uint64_t tableId) |
| Deletes a target table from the database. More...
|
|
|
void | beginNoLock () |
| Disables database locking by starting a new SQL transaction. More...
|
|
void | endNoLock () |
| Re-enables database locking by ending the previous SQL transaction. More...
|
|
|
bool | isTableEmpty (const std::string &tableName) |
| Checks whether a table in the database is empty. More...
|
|
bool | isTableExists (const std::string &tableName) |
| Checks whether a table exists in the database. More...
|
|
bool | isColumnExists (const std::string &tableName, const std::string &columnName) |
| Checks whether a table in the database contains a specific column. More...
|
|
std::string | getColumnType (const std::string &tableName, const std::string &columnName) |
| Gets the type of a specific table column from the database. More...
|
|
|
static std::uint64_t | getRequestCounter () |
| Gets the number of SQL requests performed since the start of the application. More...
|
|
|
void | checkConnection () |
| Checks whether the connection to the database is still valid and tries to reconnect if necessary. More...
|
|
|
static void | sqlException (const std::string &function, const sql::SQLException &e) |
| Catches a SQL exception and re-throws it as a specific or a generic Database::Exception. More...
|
|
|
static bool | sqlExecute (sql::PreparedStatement &sqlPreparedStatement) |
| Executes a prepared SQL statement. More...
|
|
static sql::ResultSet * | sqlExecuteQuery (sql::PreparedStatement &sqlPreparedStatement) |
| Executes a prepared SQL statement and returns the resulting set. More...
|
|
static int | sqlExecuteUpdate (sql::PreparedStatement &sqlPreparedStatement) |
| Executes a prepared SQL statement and returns the number of affected rows. More...
|
|
◆ Database() [1/3]
Constructor setting the database connection.
- Parameters
-
dbThread | Reference to the database connection used by the thread. |
◆ ~Database()
virtual crawlservpp::Wrapper::Database::~Database |
( |
| ) |
|
|
virtualdefault |
◆ Database() [2/3]
crawlservpp::Wrapper::Database::Database |
( |
Database & |
| ) |
|
|
delete |
Deleted copy constructor.
◆ Database() [3/3]
crawlservpp::Wrapper::Database::Database |
( |
Database && |
| ) |
|
|
delete |
Deleted move constructor.
◆ addColumn()
void crawlservpp::Wrapper::Database::addColumn |
( |
const std::string & |
tableName, |
|
|
const TableColumn & |
column |
|
) |
| |
|
inlineprotected |
Adds a column to a table in the database.
- Parameters
-
tableName | Constant reference to a string containing the name of the table to which the column will be added. |
column | Constant reference to a structure containing the properties of the column to be added to the table. |
- Exceptions
-
Main::Database::Exception | if no table, column, or column type has been specified, i.e. if one of the strings containing the name of the table, the name of the column, and the type of the column is empty, or if a column reference is incomplete. |
Main::Database::Exception | if a MySQL error occured while adding the column to the given table in the database. |
- See also
- Struct::TableColumn
References crawlservpp::Main::Database::addColumn(), and database.
Referenced by addTargetColumn().
◆ addDatabaseLock()
void crawlservpp::Wrapper::Database::addDatabaseLock |
( |
const std::string & |
name, |
|
|
const IsRunningCallback & |
isRunningCallback |
|
) |
| |
|
inlinestaticprotected |
Adds a lock to the database class, blocking execution.
If a lock with the same name already exists, the function will block execution until this lock has been released, or the specified callback function returns false.
- Parameters
-
name | Constant reference to a string containing the name of the lock to be waited for and added to the database class. |
isRunningCallback | Constant reference to a function that will be regularly called during a block, to enquire whether the thread (or application) is still running. As soon as this function returns false, execution will no longer be blocked, even if the lock could not be added. |
References crawlservpp::Main::Database::addDatabaseLock().
◆ addOrUpdateTargetTable()
std::uint64_t crawlservpp::Wrapper::Database::addOrUpdateTargetTable |
( |
const TargetTableProperties & |
properties | ) |
|
|
inline |
Adds a new target table or updates an existing target table in the database.
- Parameters
-
properties | Constant reference to the properties of the new target table, or the existing target table to be updated. |
- Returns
- If no target table with the specified type and name already exists, a unique ID identifying the new target table in the database. The ID of the new table is, however, only unique among all target tables of the same type. If a target table with the specified type and name already exists, its ID will be returned instead.
- Exceptions
-
Main::Database::Exception | if no type, website, URL list, name, or columns have been specfied in the given properties of the new target table, or if a column of the already existing target table cannot be overwritten due to incompatibilities between the respective data types. |
Main::Database::Exception | if a MySQL error occured while adding the new target table, or updating the existing target table in the database. |
References crawlservpp::Main::Database::addOrUpdateTargetTable(), and database.
Referenced by crawlservpp::Module::Analyzer::Database::addAdditionalTable(), crawlservpp::Module::Parser::Database::initTargetTable(), crawlservpp::Module::Analyzer::Database::initTargetTable(), and crawlservpp::Module::Extractor::Database::initTargetTables().
◆ addPreparedStatement()
void crawlservpp::Wrapper::Database::addPreparedStatement |
( |
const std::string & |
sqlQuery, |
|
|
std::size_t & |
id |
|
) |
| |
|
inlineprotected |
◆ addTargetColumn()
void crawlservpp::Wrapper::Database::addTargetColumn |
( |
const std::string & |
tableName, |
|
|
const TableColumn & |
column |
|
) |
| |
|
inline |
◆ beginNoLock()
void crawlservpp::Wrapper::Database::beginNoLock |
( |
| ) |
|
|
inline |
◆ checkConnection()
void crawlservpp::Wrapper::Database::checkConnection |
( |
| ) |
|
|
inlineprotected |
Checks whether the connection to the database is still valid and tries to reconnect if necessary.
- Warning
- Afterwards, old references to prepared SQL statements might be invalid, because the connection to the database might have been reset.
- Exceptions
-
References crawlservpp::Main::Database::checkConnection(), and database.
Referenced by crawlservpp::Module::Crawler::Database::addUrlIfNotExists(), crawlservpp::Module::Crawler::Database::addUrlsIfNotExist(), crawlservpp::Module::Extractor::Database::checkExtractingTable(), crawlservpp::Module::Parser::Database::checkParsingTable(), crawlservpp::Module::Analyzer::Database::checkSources(), crawlservpp::Module::Parser::Database::fetchUrls(), crawlservpp::Module::Extractor::Database::fetchUrls(), crawlservpp::Module::Parser::Database::getAllContents(), crawlservpp::Module::Extractor::Database::getContent(), crawlservpp::Module::Parser::Database::getContentIdFromParsedId(), crawlservpp::Module::Parser::Database::getLatestContent(), crawlservpp::Module::Extractor::Database::getLatestParsedData(), crawlservpp::Module::Parser::Database::getLockTime(), crawlservpp::Module::Extractor::Database::getLockTime(), crawlservpp::Module::Crawler::Database::getNextUrl(), crawlservpp::Module::Parser::Database::getNumberOfContents(), crawlservpp::Module::Crawler::Database::getNumberOfUrls(), crawlservpp::Module::Parser::Database::getNumberOfUrls(), crawlservpp::Module::Extractor::Database::getNumberOfUrls(), crawlservpp::Module::Analyzer::Database::getTargetTableUpdated(), crawlservpp::Module::Crawler::Database::getUrlId(), crawlservpp::Module::Crawler::Database::getUrlLockTime(), crawlservpp::Module::Parser::Database::getUrlLockTime(), crawlservpp::Module::Extractor::Database::getUrlLockTime(), crawlservpp::Module::Crawler::Database::getUrlPosition(), crawlservpp::Module::Parser::Database::getUrlPosition(), crawlservpp::Module::Extractor::Database::getUrlPosition(), crawlservpp::Module::Crawler::Database::isArchivedContentExists(), crawlservpp::Module::Crawler::Database::isUrlCrawled(), crawlservpp::Module::Crawler::Database::lockUrlIfOk(), crawlservpp::Module::Crawler::Database::prepare(), crawlservpp::Module::Parser::Database::prepare(), crawlservpp::Module::Extractor::Database::prepare(), crawlservpp::Module::Analyzer::Database::prepare(), crawlservpp::Module::Parser::Database::renewUrlLockIfOk(), crawlservpp::Module::Extractor::Database::renewUrlLockIfOk(), crawlservpp::Module::Crawler::Database::saveArchivedContent(), crawlservpp::Module::Crawler::Database::saveContent(), crawlservpp::Module::Crawler::Database::setUrlFinishedIfOk(), crawlservpp::Module::Parser::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Extractor::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Crawler::Database::unLockUrlIfOk(), crawlservpp::Module::Parser::Database::unLockUrlIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlIfOk(), crawlservpp::Module::Parser::Database::unLockUrlsIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlsIfOk(), crawlservpp::Module::Analyzer::Database::updateAdditionalTable(), crawlservpp::Module::Parser::Database::updateOrAddEntries(), crawlservpp::Module::Extractor::Database::updateOrAddEntries(), crawlservpp::Module::Extractor::Database::updateOrAddLinked(), crawlservpp::Module::Parser::Database::updateTargetTable(), crawlservpp::Module::Analyzer::Database::updateTargetTable(), crawlservpp::Module::Extractor::Database::updateTargetTable(), crawlservpp::Module::Crawler::Database::urlDuplicationCheck(), crawlservpp::Module::Crawler::Database::urlEmptyCheck(), and crawlservpp::Module::Crawler::Database::urlHashCheck().
◆ clearPreparedStatement()
void crawlservpp::Wrapper::Database::clearPreparedStatement |
( |
std::size_t & |
id | ) |
|
|
inlineprotected |
◆ compressTable()
void crawlservpp::Wrapper::Database::compressTable |
( |
const std::string & |
tableName | ) |
|
|
inlineprotected |
Compresses a table in the database.
The function will have no effect om the table, if the table is already compressed.
- Parameters
-
tableName | Constant reference to a string containing the name of the table to be compressed. |
- Exceptions
-
Main::Database::Exception | if no table is specified, i.e. if the string containing the name of the table is empty, or if a row format could not be determined. |
Main::Database::Exception | if a MySQL error occured while compressing the table in the database. |
References crawlservpp::Main::Database::compressTable(), and database.
◆ createTable()
void crawlservpp::Wrapper::Database::createTable |
( |
const TableProperties & |
properties | ) |
|
|
inlineprotected |
Adds a table to the database.
- Note
- A column for the primary key named
id
will be created automatically.
- Parameters
-
properties | Constant reference to a structure containing the properties of the table to be created. |
- Exceptions
-
Main::Database::Exception | if no name or columns are specified in the given properties structure, if one of the columns defined there is missing its name or data type, or if a column reference is incomplete. |
Main::Database::Exception | if a MySQL error occured while adding the table to the database. |
- See also
- Struct::TableProperties
References crawlservpp::Main::Database::createTable(), and database.
◆ deleteTargetTable()
void crawlservpp::Wrapper::Database::deleteTargetTable |
( |
const std::string & |
type, |
|
|
std::uint64_t |
tableId |
|
) |
| |
|
inline |
◆ dropTable()
void crawlservpp::Wrapper::Database::dropTable |
( |
const std::string & |
tableName | ) |
|
|
inlineprotected |
◆ endNoLock()
void crawlservpp::Wrapper::Database::endNoLock |
( |
| ) |
|
|
inline |
◆ getColumnType()
std::string crawlservpp::Wrapper::Database::getColumnType |
( |
const std::string & |
tableName, |
|
|
const std::string & |
columnName |
|
) |
| |
|
inline |
Gets the type of a specific table column from the database.
- Parameters
-
tableName | Constant reference to a string containing the name of the table in the database from which the type of the column will be retrieved. |
columnName | Constant reference to a string containing the name of the column whose type will be retrieved. |
- Returns
- A copy of the name of the given table column's data type, without specifiers like 'UNSIGNED'.
- Exceptions
-
Main::Database::Exception | if no table or column has been specified, i.e. one of the strings containing the name and the column is empty. |
Main::Database::Exception | if a MySQL error occured while retrieving the type of the given column, e.g. if the specified table does not exist. |
References database, and crawlservpp::Main::Database::getColumnType().
Referenced by crawlservpp::Module::Analyzer::Thread::uploadResult().
◆ getConfiguration()
std::string crawlservpp::Wrapper::Database::getConfiguration |
( |
std::uint64_t |
configId | ) |
|
|
inline |
◆ getCustomData() [1/6]
void crawlservpp::Wrapper::Database::getCustomData |
( |
Data::GetValue & |
data | ) |
|
|
inline |
◆ getCustomData() [2/6]
void crawlservpp::Wrapper::Database::getCustomData |
( |
Data::GetFields & |
data | ) |
|
|
inline |
◆ getCustomData() [3/6]
◆ getCustomData() [4/6]
void crawlservpp::Wrapper::Database::getCustomData |
( |
Data::GetColumn & |
data | ) |
|
|
inline |
◆ getCustomData() [5/6]
◆ getCustomData() [6/6]
◆ getLastInsertedId()
std::uint64_t crawlservpp::Wrapper::Database::getLastInsertedId |
( |
| ) |
|
|
inlineprotected |
◆ getLoggingMin()
std::uint8_t crawlservpp::Wrapper::Database::getLoggingMin |
( |
| ) |
const |
|
inlineprotected |
Gets the minimal logging level.
- Returns
- The minimum logging level, in which logging is still not deactivated.
References database.
Referenced by crawlservpp::Module::Analyzer::Database::addAdditionalTable(), crawlservpp::Module::Analyzer::Database::checkSources(), crawlservpp::Module::Analyzer::Database::getCorpus(), crawlservpp::Module::Analyzer::Database::initTargetTable(), crawlservpp::Module::Crawler::Database::saveArchivedContent(), crawlservpp::Module::Crawler::Database::saveContent(), crawlservpp::Module::Parser::Database::setTargetTable(), crawlservpp::Module::Extractor::Database::setTargetTable(), crawlservpp::Module::Analyzer::Database::setTargetTable(), crawlservpp::Module::Analyzer::Database::updateAdditionalTable(), crawlservpp::Module::Parser::Database::updateTargetTable(), crawlservpp::Module::Analyzer::Database::updateTargetTable(), crawlservpp::Module::Extractor::Database::updateTargetTable(), and crawlservpp::Module::Crawler::Database::urlHashCheck().
◆ getLoggingVerbose()
std::uint8_t crawlservpp::Wrapper::Database::getLoggingVerbose |
( |
| ) |
const |
|
inlineprotected |
◆ getMaxAllowedPacketSize()
std::uint64_t crawlservpp::Wrapper::Database::getMaxAllowedPacketSize |
( |
| ) |
const |
|
inlineprotected |
◆ getOptions()
◆ getPreparedStatement()
sql::PreparedStatement & crawlservpp::Wrapper::Database::getPreparedStatement |
( |
std::size_t |
id | ) |
|
|
inlineprotected |
Gets a reference to a prepared SQL statement.
- Warning
- Do not run checkConnection while using this reference, because the references will be invalidated when reconnecting to the database!
- Parameters
-
id | The ID of the prepared SQL statement to retrieve. |
- Returns
- A reference to the prepared SQL statement.
- Exceptions
-
References database, and crawlservpp::Main::Database::getPreparedStatement().
Referenced by crawlservpp::Module::Crawler::Database::addUrlIfNotExists(), crawlservpp::Module::Crawler::Database::addUrlsIfNotExist(), crawlservpp::Module::Extractor::Database::checkExtractingTable(), crawlservpp::Module::Parser::Database::checkParsingTable(), crawlservpp::Module::Analyzer::Database::checkSources(), crawlservpp::Module::Parser::Database::fetchUrls(), crawlservpp::Module::Extractor::Database::fetchUrls(), crawlservpp::Module::Parser::Database::getAllContents(), crawlservpp::Module::Extractor::Database::getContent(), crawlservpp::Module::Parser::Database::getContentIdFromParsedId(), crawlservpp::Module::Parser::Database::getLatestContent(), crawlservpp::Module::Extractor::Database::getLatestParsedData(), crawlservpp::Module::Parser::Database::getLockTime(), crawlservpp::Module::Extractor::Database::getLockTime(), crawlservpp::Module::Crawler::Database::getNextUrl(), crawlservpp::Module::Parser::Database::getNumberOfContents(), crawlservpp::Module::Crawler::Database::getNumberOfUrls(), crawlservpp::Module::Parser::Database::getNumberOfUrls(), crawlservpp::Module::Extractor::Database::getNumberOfUrls(), crawlservpp::Module::Analyzer::Database::getTargetTableUpdated(), crawlservpp::Module::Crawler::Database::getUrlId(), crawlservpp::Module::Crawler::Database::getUrlLockTime(), crawlservpp::Module::Parser::Database::getUrlLockTime(), crawlservpp::Module::Extractor::Database::getUrlLockTime(), crawlservpp::Module::Crawler::Database::getUrlPosition(), crawlservpp::Module::Parser::Database::getUrlPosition(), crawlservpp::Module::Extractor::Database::getUrlPosition(), crawlservpp::Module::Crawler::Database::isArchivedContentExists(), crawlservpp::Module::Crawler::Database::isUrlCrawled(), crawlservpp::Module::Crawler::Database::lockUrlIfOk(), crawlservpp::Module::Parser::Database::renewUrlLockIfOk(), crawlservpp::Module::Extractor::Database::renewUrlLockIfOk(), crawlservpp::Module::Crawler::Database::saveArchivedContent(), crawlservpp::Module::Crawler::Database::saveContent(), crawlservpp::Module::Crawler::Database::setUrlFinishedIfOk(), crawlservpp::Module::Parser::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Extractor::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Crawler::Database::unLockUrlIfOk(), crawlservpp::Module::Parser::Database::unLockUrlIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlIfOk(), crawlservpp::Module::Parser::Database::unLockUrlsIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlsIfOk(), crawlservpp::Module::Analyzer::Database::updateAdditionalTable(), crawlservpp::Module::Parser::Database::updateOrAddEntries(), crawlservpp::Module::Extractor::Database::updateOrAddEntries(), crawlservpp::Module::Extractor::Database::updateOrAddLinked(), crawlservpp::Module::Parser::Database::updateTargetTable(), crawlservpp::Module::Analyzer::Database::updateTargetTable(), crawlservpp::Module::Extractor::Database::updateTargetTable(), crawlservpp::Module::Crawler::Database::urlDuplicationCheck(), crawlservpp::Module::Crawler::Database::urlEmptyCheck(), and crawlservpp::Module::Crawler::Database::urlHashCheck().
◆ getQueryProperties()
void crawlservpp::Wrapper::Database::getQueryProperties |
( |
std::uint64_t |
queryId, |
|
|
QueryProperties & |
queryPropertiesTo |
|
) |
| |
|
inline |
◆ getRequestCounter()
std::uint64_t crawlservpp::Wrapper::Database::getRequestCounter |
( |
| ) |
|
|
inlinestatic |
◆ getTargetTableId()
std::uint64_t crawlservpp::Wrapper::Database::getTargetTableId |
( |
const std::string & |
type, |
|
|
std::uint64_t |
listId, |
|
|
const std::string & |
tableName |
|
) |
| |
|
inline |
Gets the ID of a target table from the database.
- Parameters
-
type | Constant reference to a string containing the type of the target table for which to retrieve its ID. |
listId | The ID of the URL list associated with the target table for which to retrieve its ID. |
tableName | Const reference to a string containing the name of the target table for which to retrieve its ID. |
- Returns
- The ID of the specified target table as stored in the database.
- Exceptions
-
Main::Database::Exception | if no target table or URL list has been specified, i.e. if the string containing the type is empty, or the target table or the URL list ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the ID of the target table from the database. |
References database, and crawlservpp::Main::Database::getTargetTableId().
◆ getTargetTableName()
std::string crawlservpp::Wrapper::Database::getTargetTableName |
( |
const std::string & |
type, |
|
|
std::uint64_t |
tableId |
|
) |
| |
|
inline |
Gets the name of a target table from the database.
- Parameters
-
type | String view containing the type of the target table for which to retrieve its name. |
tableId | The ID of the target table for which to retrieve its name. |
- Returns
- A copy of the name of the specified target table as stored in the database.
- Exceptions
-
Main::Database::Exception | if no target table has been specified, i.e. the string containing the type is empty or the target table ID is zero. |
Main::Database::Exception | if a MySQL error occured while retrieving the name of the target table from the database. |
References database, and crawlservpp::Main::Database::getTargetTableName().
◆ getTargetTables()
std::queue< Database::IdString > crawlservpp::Wrapper::Database::getTargetTables |
( |
const std::string & |
type, |
|
|
std::uint64_t |
listId |
|
) |
| |
|
inline |
Gets the target tables of the specified type for a URL list from the database.
- Parameters
-
type | Constant reference to a string containing the type of the target tables to retrieve. |
listId | The ID of the URL list for which to retrieve the target tables. |
- Returns
- A queue containing the IDs and names of the target tables of the given type for the specified URL list.
- Exceptions
-
References database, and crawlservpp::Main::Database::getTargetTables().
◆ getUrlListIdString()
const std::string & crawlservpp::Wrapper::Database::getUrlListIdString |
( |
| ) |
const |
|
inlineprotected |
◆ getWebsiteDomain()
std::string crawlservpp::Wrapper::Database::getWebsiteDomain |
( |
std::uint64_t |
websiteId | ) |
|
|
inline |
◆ getWebsiteIdString()
const std::string & crawlservpp::Wrapper::Database::getWebsiteIdString |
( |
| ) |
const |
|
inlineprotected |
◆ insertCustomData() [1/3]
void crawlservpp::Wrapper::Database::insertCustomData |
( |
const Data::InsertValue & |
data | ) |
|
|
inline |
Inserts a custom value into a table row in the database.
- Parameters
-
data | Constant reference to a structure containing the data to be inserted. |
- Exceptions
-
Main::Database::Exception | if no table, columns, or column type have been specified in the given data structure, if the given data is too large, or if an invalid data has been encountered. |
Main::Database::Exception | if a MySQL error occured while inserting the data. |
- See also
- Data::InsertValue
References database, and crawlservpp::Main::Database::insertCustomData().
Referenced by crawlservpp::Module::Analyzer::Algo::CorpusGenerator::onAlgoInit(), crawlservpp::Module::Analyzer::Algo::ExtractIds::resetAlgo(), crawlservpp::Module::Analyzer::Algo::WordsOverTime::resetAlgo(), crawlservpp::Module::Analyzer::Algo::Assoc::resetAlgo(), crawlservpp::Module::Analyzer::Algo::AssocOverTime::resetAlgo(), crawlservpp::Module::Analyzer::Algo::AllTokens::resetAlgo(), crawlservpp::Module::Analyzer::Algo::SentimentOverTime::resetAlgo(), and crawlservpp::Module::Analyzer::Algo::TopicModelling::resetAlgo().
◆ insertCustomData() [2/3]
◆ insertCustomData() [3/3]
◆ isColumnExists()
bool crawlservpp::Wrapper::Database::isColumnExists |
( |
const std::string & |
tableName, |
|
|
const std::string & |
columnName |
|
) |
| |
|
inline |
Checks whether a table in the database contains a specific column.
- Parameters
-
tableName | Constant reference to a string containing the name of the table in the database in which the existence of the column will be checked. |
columnName | Constant reference to a string containing the name of the column to be checked for in the given table. |
- Returns
- True, if the given column exists in the specified table. False otherwise.
- Exceptions
-
Main::Database::Exception | if no table or columns have been specified, i.e. one of the strings containing the name and the column is empty. |
Main::Database::Exception | if a MySQL error occured while checking the existence of the given column, e.g. if the specified table does not exist. |
References database, and crawlservpp::Main::Database::isColumnExists().
Referenced by addTargetColumn(), and crawlservpp::Module::Analyzer::Thread::uploadResult().
◆ isTableEmpty()
bool crawlservpp::Wrapper::Database::isTableEmpty |
( |
const std::string & |
tableName | ) |
|
|
inline |
Checks whether a table in the database is empty.
- Parameters
-
tableName | Constant reference to a string containing the name of the table whose contents will be checked in the database. |
- Returns
- True, if the given table is empty. False if it contains data.
- Exceptions
-
References database, and crawlservpp::Main::Database::isTableEmpty().
◆ isTableExists()
bool crawlservpp::Wrapper::Database::isTableExists |
( |
const std::string & |
tableName | ) |
|
|
inline |
Checks whether a table exists in the database.
- Parameters
-
tableName | Constant reference to a string containing the name of the table whose existence in the database will be checked. |
- Returns
- True, if the given table exists in the database. False otherwise.
- Exceptions
-
References database, and crawlservpp::Main::Database::isTableExists().
◆ log() [1/2]
void crawlservpp::Wrapper::Database::log |
( |
std::uint8_t |
level, |
|
|
const std::string & |
logEntry |
|
) |
| |
|
inline |
Writes a thread-specific log entry to the database.
Removes invalid UTF-8 characters if necessary.
If debug logging is active, the entry will be written to the logging file as well.
The log entry will not be written to the database, if the current logging level is lower than the specified logging level. The logging level does not affect the writing of logging entries being to the logging file when debug logging is active.
- Note
- String views cannot be used, because they are not supported by the API for the MySQL database.
- Parameters
-
level | The logging level for the entry. The entry will only be written to the database, if the current logging level is at least the logging level for the entry. |
logEntry | Constant reference to a string containing the log entry. |
References database, and crawlservpp::Module::Database::log().
Referenced by crawlservpp::Module::Analyzer::Database::addAdditionalTable(), crawlservpp::Module::Analyzer::Database::checkSources(), crawlservpp::Module::Analyzer::Database::getCorpus(), crawlservpp::Module::Analyzer::Database::initTargetTable(), crawlservpp::Module::Crawler::Database::prepare(), crawlservpp::Module::Parser::Database::prepare(), crawlservpp::Module::Extractor::Database::prepare(), crawlservpp::Module::Analyzer::Database::prepare(), crawlservpp::Module::Crawler::Database::saveArchivedContent(), crawlservpp::Module::Crawler::Database::saveContent(), crawlservpp::Module::Parser::Database::setTargetTable(), crawlservpp::Module::Extractor::Database::setTargetTable(), crawlservpp::Module::Analyzer::Database::setTargetTable(), crawlservpp::Module::Analyzer::Database::updateAdditionalTable(), crawlservpp::Module::Parser::Database::updateTargetTable(), crawlservpp::Module::Analyzer::Database::updateTargetTable(), crawlservpp::Module::Extractor::Database::updateTargetTable(), and crawlservpp::Module::Crawler::Database::urlHashCheck().
◆ log() [2/2]
void crawlservpp::Wrapper::Database::log |
( |
std::uint8_t |
level, |
|
|
std::queue< std::string > & |
logEntries |
|
) |
| |
|
inline |
Writes multiple thread-specific log entries to the database.
Removes invalid UTF-8 characters if necessary.
If debug logging is active, the entries will be written to the logging file as well.
The log entries will not be written to the database, if the current logging level is lower than the specified logging level. The logging level does not affect the writing of logging entries being to the logging file when debug logging is active.
- Note
- String views cannot be used, because they are not supported by the API for the MySQL database.
- Parameters
-
level | The logging level for the entries. The entries will only be written to the database, if the current logging level is at least the logging level for the entry. |
logEntries | Reference to a queue of strings containing the log entries to be written. It will be emptied regardless whether the log entries will be written to the database. |
References database, and crawlservpp::Module::Database::log().
◆ operator=() [1/2]
Deleted copy assignment operator.
◆ operator=() [2/2]
Deleted move assignment operator.
◆ removeDatabaseLock()
void crawlservpp::Wrapper::Database::removeDatabaseLock |
( |
const std::string & |
name | ) |
|
|
inlinestaticprotected |
Removes a lock from the database class.
Does nothing if a lock with the given name does not exist in the database class.
- Parameters
-
name | Constant reference to a string containing the name of the lock to be removed from the database class. |
References crawlservpp::Main::Database::removeDatabaseLock().
◆ reserveForPreparedStatements()
void crawlservpp::Wrapper::Database::reserveForPreparedStatements |
( |
std::size_t |
n | ) |
|
|
inlineprotected |
◆ setLogging()
void crawlservpp::Wrapper::Database::setLogging |
( |
std::uint8_t |
level, |
|
|
std::uint8_t |
min, |
|
|
std::uint8_t |
verbose |
|
) |
| |
|
inline |
◆ setSleepOnError()
void crawlservpp::Wrapper::Database::setSleepOnError |
( |
std::uint64_t |
seconds | ) |
|
|
inline |
◆ setTimeOut()
void crawlservpp::Wrapper::Database::setTimeOut |
( |
std::uint64_t |
milliseconds | ) |
|
|
inline |
◆ setUrlListCaseSensitive()
void crawlservpp::Wrapper::Database::setUrlListCaseSensitive |
( |
std::uint64_t |
listId, |
|
|
bool |
isCaseSensitive |
|
) |
| |
|
inlineprotected |
◆ sqlException()
void crawlservpp::Wrapper::Database::sqlException |
( |
const std::string & |
function, |
|
|
const sql::SQLException & |
e |
|
) |
| |
|
inlinestaticprotected |
Catches a SQL exception and re-throws it as a specific or a generic Database::Exception.
- Note
- Always throws an exception.
- Parameters
-
function | Constant reference to a string containing the name of the function in which the exception has been thrown. |
e | Constant reference to the SQL exception that has been thrown. |
- Exceptions
-
References crawlservpp::Main::Database::sqlException().
Referenced by crawlservpp::Module::Crawler::Database::addUrlIfNotExists(), crawlservpp::Module::Crawler::Database::addUrlsIfNotExist(), crawlservpp::Module::Extractor::Database::checkExtractingTable(), crawlservpp::Module::Parser::Database::checkParsingTable(), crawlservpp::Module::Analyzer::Database::checkSources(), crawlservpp::Module::Parser::Database::fetchUrls(), crawlservpp::Module::Extractor::Database::fetchUrls(), crawlservpp::Module::Parser::Database::getAllContents(), crawlservpp::Module::Extractor::Database::getContent(), crawlservpp::Module::Parser::Database::getContentIdFromParsedId(), crawlservpp::Module::Parser::Database::getLatestContent(), crawlservpp::Module::Extractor::Database::getLatestParsedData(), crawlservpp::Module::Parser::Database::getLockTime(), crawlservpp::Module::Extractor::Database::getLockTime(), crawlservpp::Module::Crawler::Database::getNextUrl(), crawlservpp::Module::Parser::Database::getNumberOfContents(), crawlservpp::Module::Crawler::Database::getNumberOfUrls(), crawlservpp::Module::Parser::Database::getNumberOfUrls(), crawlservpp::Module::Extractor::Database::getNumberOfUrls(), crawlservpp::Module::Analyzer::Database::getTargetTableUpdated(), crawlservpp::Module::Crawler::Database::getUrlId(), crawlservpp::Module::Crawler::Database::getUrlLockTime(), crawlservpp::Module::Parser::Database::getUrlLockTime(), crawlservpp::Module::Extractor::Database::getUrlLockTime(), crawlservpp::Module::Crawler::Database::getUrlPosition(), crawlservpp::Module::Parser::Database::getUrlPosition(), crawlservpp::Module::Extractor::Database::getUrlPosition(), crawlservpp::Module::Crawler::Database::isArchivedContentExists(), crawlservpp::Module::Crawler::Database::isUrlCrawled(), crawlservpp::Module::Crawler::Database::lockUrlIfOk(), crawlservpp::Module::Analyzer::Database::prepare(), crawlservpp::Module::Parser::Database::renewUrlLockIfOk(), crawlservpp::Module::Extractor::Database::renewUrlLockIfOk(), crawlservpp::Module::Crawler::Database::saveArchivedContent(), crawlservpp::Module::Crawler::Database::saveContent(), crawlservpp::Module::Crawler::Database::setUrlFinishedIfOk(), crawlservpp::Module::Parser::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Extractor::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Crawler::Database::unLockUrlIfOk(), crawlservpp::Module::Parser::Database::unLockUrlIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlIfOk(), crawlservpp::Module::Parser::Database::unLockUrlsIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlsIfOk(), crawlservpp::Module::Analyzer::Database::updateAdditionalTable(), crawlservpp::Module::Parser::Database::updateOrAddEntries(), crawlservpp::Module::Extractor::Database::updateOrAddEntries(), crawlservpp::Module::Extractor::Database::updateOrAddLinked(), crawlservpp::Module::Parser::Database::updateTargetTable(), crawlservpp::Module::Analyzer::Database::updateTargetTable(), crawlservpp::Module::Extractor::Database::updateTargetTable(), crawlservpp::Module::Crawler::Database::urlDuplicationCheck(), crawlservpp::Module::Crawler::Database::urlEmptyCheck(), and crawlservpp::Module::Crawler::Database::urlHashCheck().
◆ sqlExecute()
bool crawlservpp::Wrapper::Database::sqlExecute |
( |
sql::PreparedStatement & |
sqlPreparedStatement | ) |
|
|
inlinestaticprotected |
Executes a prepared SQL statement.
- Parameters
-
sqlPreparedStatement | Reference to the prepared SQL statement to be executed. |
- Returns
- True, if the prepared SQL statement returned a result set. False, if the statement returned nothing or an update count.
References crawlservpp::Main::Database::sqlExecute().
Referenced by crawlservpp::Module::Analyzer::Database::checkSources(), crawlservpp::Module::Parser::Database::fetchUrls(), crawlservpp::Module::Extractor::Database::fetchUrls(), crawlservpp::Module::Crawler::Database::saveArchivedContent(), crawlservpp::Module::Crawler::Database::saveContent(), crawlservpp::Module::Crawler::Database::setUrlFinishedIfOk(), crawlservpp::Module::Parser::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Extractor::Database::setUrlsFinishedIfLockOk(), crawlservpp::Module::Crawler::Database::unLockUrlIfOk(), crawlservpp::Module::Parser::Database::unLockUrlsIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlsIfOk(), crawlservpp::Module::Parser::Database::updateOrAddEntries(), crawlservpp::Module::Extractor::Database::updateOrAddEntries(), and crawlservpp::Module::Extractor::Database::updateOrAddLinked().
◆ sqlExecuteQuery()
sql::ResultSet * crawlservpp::Wrapper::Database::sqlExecuteQuery |
( |
sql::PreparedStatement & |
sqlPreparedStatement | ) |
|
|
inlinestaticprotected |
Executes a prepared SQL statement and returns the resulting set.
- Parameters
-
sqlPreparedStatement | Reference to the prepared SQL statement to be executed. |
- Returns
- A pointer to the result set retrieved by executing the prepared SQL statement.
References crawlservpp::Main::Database::sqlExecuteQuery().
Referenced by crawlservpp::Module::Analyzer::Database::checkSources(), crawlservpp::Module::Parser::Database::fetchUrls(), crawlservpp::Module::Extractor::Database::fetchUrls(), crawlservpp::Module::Parser::Database::getAllContents(), crawlservpp::Module::Extractor::Database::getContent(), crawlservpp::Module::Parser::Database::getContentIdFromParsedId(), crawlservpp::Module::Parser::Database::getLatestContent(), crawlservpp::Module::Extractor::Database::getLatestParsedData(), crawlservpp::Module::Parser::Database::getLockTime(), crawlservpp::Module::Extractor::Database::getLockTime(), crawlservpp::Module::Crawler::Database::getNextUrl(), crawlservpp::Module::Parser::Database::getNumberOfContents(), crawlservpp::Module::Crawler::Database::getNumberOfUrls(), crawlservpp::Module::Parser::Database::getNumberOfUrls(), crawlservpp::Module::Extractor::Database::getNumberOfUrls(), crawlservpp::Module::Analyzer::Database::getTargetTableUpdated(), crawlservpp::Module::Crawler::Database::getUrlId(), crawlservpp::Module::Crawler::Database::getUrlLockTime(), crawlservpp::Module::Parser::Database::getUrlLockTime(), crawlservpp::Module::Extractor::Database::getUrlLockTime(), crawlservpp::Module::Crawler::Database::getUrlPosition(), crawlservpp::Module::Parser::Database::getUrlPosition(), crawlservpp::Module::Extractor::Database::getUrlPosition(), crawlservpp::Module::Crawler::Database::isArchivedContentExists(), crawlservpp::Module::Crawler::Database::isUrlCrawled(), crawlservpp::Module::Crawler::Database::urlDuplicationCheck(), crawlservpp::Module::Crawler::Database::urlEmptyCheck(), and crawlservpp::Module::Crawler::Database::urlHashCheck().
◆ sqlExecuteUpdate()
int crawlservpp::Wrapper::Database::sqlExecuteUpdate |
( |
sql::PreparedStatement & |
sqlPreparedStatement | ) |
|
|
inlinestaticprotected |
Executes a prepared SQL statement and returns the number of affected rows.
- Parameters
-
sqlPreparedStatement | Reference to the prepared SQL statement to be executed. |
- Returns
- The number of rows affected by the prepared SQL statement
References crawlservpp::Main::Database::sqlExecuteUpdate().
Referenced by crawlservpp::Module::Crawler::Database::addUrlIfNotExists(), crawlservpp::Module::Crawler::Database::addUrlsIfNotExist(), crawlservpp::Module::Extractor::Database::checkExtractingTable(), crawlservpp::Module::Parser::Database::checkParsingTable(), crawlservpp::Module::Crawler::Database::isArchivedContentExists(), crawlservpp::Module::Crawler::Database::lockUrlIfOk(), crawlservpp::Module::Parser::Database::renewUrlLockIfOk(), crawlservpp::Module::Extractor::Database::renewUrlLockIfOk(), crawlservpp::Module::Parser::Database::unLockUrlIfOk(), crawlservpp::Module::Extractor::Database::unLockUrlIfOk(), crawlservpp::Module::Analyzer::Database::updateAdditionalTable(), crawlservpp::Module::Parser::Database::updateTargetTable(), crawlservpp::Module::Analyzer::Database::updateTargetTable(), crawlservpp::Module::Extractor::Database::updateTargetTable(), and crawlservpp::Module::Crawler::Database::urlHashCheck().
◆ tryDatabaseLock()
bool crawlservpp::Wrapper::Database::tryDatabaseLock |
( |
const std::string & |
name | ) |
|
|
inlinestaticprotected |
Tries to add a lock to the database class, not blocking execution.
If a lock with the same name already exists, the function will not add a lock and return false instead.
- Parameters
-
name | Constant reference to a string containing the name of the lock to be added to the database class if a lock with the same name does not exist already. |
- Returns
- True, if a lock with the same name did not exist already and the lock has been added. False, if a lock with the same name already exists and no lock has been added.
References crawlservpp::Main::Database::tryDatabaseLock().
◆ updateCustomData() [1/3]
void crawlservpp::Wrapper::Database::updateCustomData |
( |
const Data::UpdateValue & |
data | ) |
|
|
inline |
◆ updateCustomData() [2/3]
◆ updateCustomData() [3/3]
◆ Wrapper::DatabaseLock
Allows scoped locking of the database.
◆ Wrapper::DatabaseTryLock
Allows optional scoped locking of the database.
◆ database
Reference to the database connection for the thread.
Referenced by addColumn(), addOrUpdateTargetTable(), addPreparedStatement(), beginNoLock(), checkConnection(), crawlservpp::Module::Analyzer::Database::checkSources(), clearPreparedStatement(), compressTable(), createTable(), deleteTargetTable(), dropTable(), endNoLock(), getColumnType(), getConfiguration(), getCustomData(), getLastInsertedId(), getLoggingMin(), getLoggingVerbose(), getMaxAllowedPacketSize(), getOptions(), getPreparedStatement(), getQueryProperties(), getTargetTableId(), getTargetTableName(), getTargetTables(), getUrlListIdString(), getWebsiteDomain(), getWebsiteIdString(), insertCustomData(), isColumnExists(), isTableEmpty(), isTableExists(), log(), reserveForPreparedStatements(), setLogging(), setSleepOnError(), setTimeOut(), setUrlListCaseSensitive(), and updateCustomData().
The documentation for this class was generated from the following file: