32 #ifndef MODULE_EXTRACTOR_DATABASE_HPP_ 33 #define MODULE_EXTRACTOR_DATABASE_HPP_ 37 #include "../../Helper/CommaLocale.hpp" 38 #include "../../Main/Exception.hpp" 39 #include "../../Struct/DataEntry.hpp" 40 #include "../../Struct/StatusSetter.hpp" 41 #include "../../Struct/TableColumn.hpp" 42 #include "../../Struct/TargetTableProperties.hpp" 43 #include "../../Wrapper/Database.hpp" 45 #include "../../Helper/Portability/mysqlcppconn.h" 47 #include <cppconn/exception.h> 48 #include <cppconn/prepared_statement.h> 49 #include <cppconn/resultset.h> 50 #include <cppconn/statement.h> 51 #include <mysql_connection.h> 61 #include <string_view> 71 using std::string_view_literals::operator
""sv;
169 using IdString = std::pair<std::uint64_t, std::string>;
170 using StringString = std::pair<std::string, std::string>;
171 using SqlResultSetPtr = std::unique_ptr<sql::ResultSet>;
188 void setSources(std::queue<StringString>& tablesAndColumns);
213 [[nodiscard]] std::string
fetchUrls(std::uint64_t lastId, std::queue<IdString>& cache, std::uint32_t lockTimeout);
221 [[nodiscard]] std::string
getLockTime(std::uint32_t lockTimeout);
223 [[nodiscard]] std::string
renewUrlLockIfOk(std::uint64_t urlId,
const std::string& lockTime, std::uint32_t lockTimeout);
224 bool unLockUrlIfOk(std::uint64_t urlId,
const std::string& lockTime);
225 void unLockUrlsIfOk(std::queue<IdString>& urls, std::string& lockTime);
232 bool getContent(std::uint64_t urlId, IdString& contentTo);
233 void getLatestParsedData(std::uint64_t urlId, std::size_t sourceIndex, std::string& resultTo);
246 bool reExtract{
false};
247 bool extractCustom{
false};
248 std::string targetTableName;
249 std::string linkedTableName;
250 std::vector<std::string> targetFieldNames;
251 std::vector<std::string> linkedFieldNames;
252 bool overwrite{
true};
253 bool overwriteLinked{
true};
257 bool rawContentIsSource{
false};
258 std::queue<StringString> sources;
261 std::string urlListTable;
262 std::string extractingTable;
263 std::uint64_t targetTableId{};
264 std::uint64_t linkedTableId{};
265 std::string targetTableFull;
266 std::string linkedTableFull;
267 std::string linkedField;
268 std::uint64_t linkedIndex{};
272 std::size_t fetchUrls{};
273 std::size_t lockUrl{};
274 std::size_t lock10Urls{};
275 std::size_t lock100Urls{};
276 std::size_t lockMaxUrls{};
277 std::size_t getUrlPosition{};
278 std::size_t getNumberOfUrls{};
279 std::size_t getLockTime{};
280 std::size_t getUrlLockTime{};
281 std::size_t renewUrlLockIfOk{};
282 std::size_t unLockUrlIfOk{};
283 std::size_t checkExtractingTable{};
284 std::size_t getContent{};
285 std::size_t updateOrAddEntry{};
286 std::size_t updateOrAddLinked{};
287 std::size_t updateOrAdd10Entries{};
288 std::size_t updateOrAdd10Linked{};
289 std::size_t updateOrAdd100Entries{};
290 std::size_t updateOrAdd100Linked{};
291 std::size_t updateOrAddMaxEntries{};
292 std::size_t updateOrAddMaxLinked{};
293 std::size_t setUrlFinishedIfLockOk{};
294 std::size_t set10UrlsFinishedIfLockOk{};
295 std::size_t set100UrlsFinishedIfLockOk{};
296 std::size_t setMaxUrlsFinishedIfLockOk{};
297 std::size_t updateTargetTable{};
301 std::vector<std::size_t> psGetLatestParsedData;
304 bool checkEntrySize(DataEntry& entry);
305 [[nodiscard]] std::string queryLockUrls(std::size_t numberOfUrls);
306 [[nodiscard]] std::string queryUpdateOrAddEntries(std::size_t numberOfEntries);
307 [[nodiscard]] std::string queryUpdateOrAddLinked(std::size_t numberOfEntries);
308 [[nodiscard]] std::string querySetUrlsFinishedIfLockOk(std::size_t numberOfUrls);
309 [[nodiscard]] std::string queryUnlockUrlsIfOk(std::size_t numberOfUrls);
A data entry containing either parsed or extracted data.
Definition: DataEntry.hpp:45
Target table properties containing its type, website, URL list, table names, columns, and compression.
Definition: TargetTableProperties.hpp:44
Class handling database access for threads.
Definition: Database.hpp:91
#define MAIN_EXCEPTION_CLASS()
Macro used to easily define classes for general exceptions.
Definition: Exception.hpp:50
Structure for table columns containing its name, type, reference, and indexing.
Definition: TableColumn.hpp:39
Structure containing all the data needed to keep the status of a thread updated.
Definition: StatusSetter.hpp:57
Wrapper class providing the database functionality of Module::Database to its child classes...
Definition: Database.hpp:72