34 #ifndef NETWORK_CONFIG_HPP_ 35 #define NETWORK_CONFIG_HPP_ 37 #include "../Module/Config.hpp" 41 #include <string_view> 51 using std::string_view_literals::operator
""sv;
668 void reset()
override = 0;
674 [[nodiscard]]
const std::string&
getProtocol()
const;
757 bool insecure{
false};
759 this->
option(
"insecure", insecure);
762 this->
warning(
"Using INSECURE connections.");
constexpr std::uint64_t defaultTimeOutRequest
Default request time-out, in seconds.
Definition: Config.hpp:102
std::uint64_t resetTorAfter
Number of seconds until automatically using the TOR control server to request a new identity...
Definition: Config.hpp:464
bool encodingIdentity
Specifies whether to (also) request non-compressed encoding for requested content.
Definition: Config.hpp:283
std::vector< std::string > headers
Custom HTTP headers to be sent with every request.
Definition: Config.hpp:301
std::vector< std::string > http200Aliases
Aliases that will be treated like HTTP/1.0 200 OK.
Definition: Config.hpp:304
bool cookiesSession
Specifies whether to ignore obsolete session cookies.
Definition: Config.hpp:187
constexpr std::uint16_t httpVersion2
Attempt to use HTTP/2, fall back to HTTP/1.1.
Definition: Config.hpp:66
void option(const std::string &name, bool &target)
Checks for a configuration option of type bool.
Definition: Config.hpp:573
std::uint64_t speedUpLimit
Maximum upload speed in bytes per second.
Definition: Config.hpp:499
std::string proxyAuth
Authentification for the proxy server used.
Definition: Config.hpp:367
bool sslVerifyProxyHost
Specifies whether to verify that the SSL certificate is for the proxy server it is known as...
Definition: Config.hpp:520
std::string tlsSrpPassword
User name used for TLS-SRP authentification.
Definition: Config.hpp:615
std::uint16_t httpVersion
HTTP version(s) to be used.
Definition: Config.hpp:313
std::string protocol
The protocol to be used for HTTP requests.
Definition: Config.hpp:637
std::string proxy
Proxy server used.
Definition: Config.hpp:356
std::uint64_t speedDownLimit
Maximum download speed in bytes per second.
Definition: Config.hpp:476
std::int64_t dnsCacheTimeOut
The lifetime of DNS cache entries.
Definition: Config.hpp:203
constexpr std::uint16_t httpVersion3Only
Use HTTP/3 only.
Definition: Config.hpp:78
constexpr std::uint16_t httpVersion2Tls
Attempt to use HTTP/2 over TLS, fall back to HTTP/1.1.
Definition: Config.hpp:72
std::string localInterface
Interface to be used for outgoing traffic.
Definition: Config.hpp:319
bool dnsShuffle
Specifies whether to shuffle addresses when a host name returns more than one.
Definition: Config.hpp:235
std::string dnsDoH
The URL of a custom DNS-over-HTTPS (DoH) server.
Definition: Config.hpp:209
constexpr std::uint64_t defaultRedirectMax
Default maximum number of automatic redirects by default.
Definition: Config.hpp:87
std::uint16_t localPort
Port to be used for outgoing traffic.
Definition: Config.hpp:327
struct crawlservpp::Network::Config::Entries networkConfig
Configuration for networking.
const std::string & getProtocol() const
Gets the protocol to be used for networking.
Definition: Config.hpp:787
std::string tlsSrpUser
Password used for TLS-SRP authentification.
Definition: Config.hpp:606
std::string referer
The HTTP Referer header to be set.
Definition: Config.hpp:440
Abstract class as base for module-specific configurations.
Definition: Config.hpp:122
Abstract class containing the network-specific configuration for threads.
Definition: Config.hpp:121
bool contentLengthIgnore
Specifies whether the Content-Length header in HTTP responses will be ignored.
Definition: Config.hpp:140
bool encodingZstd
Specifies whether to request Zstandard encoding for requested content.
Definition: Config.hpp:298
bool tcpNagle
Specifies whether the TCP's Nagle algorithm is enabled on this connection.
Definition: Config.hpp:568
bool sslVerifyHost
Specifies whether to verify that the SSL certificate is for the server it is known as...
Definition: Config.hpp:506
bool sslVerifyPeer
Specifies whether to verify the authenticity of the server's SSL certificate.
Definition: Config.hpp:513
bool redirect
Specifies whether to follow HTTP Location headers for automatic redirects.
Definition: Config.hpp:405
constexpr std::uint16_t httpVersion2Only
Use non-TLS HTTP/2, even if HTTPS is not available.
Definition: Config.hpp:69
std::uint64_t speedLowLimit
Low speed limit in bytes per second.
Definition: Config.hpp:484
std::string proxyPre
Pre-proxy server to be used.
Definition: Config.hpp:379
void reset() override=0
Resets additional configuration options.
bool tcpFastOpen
Specifies whether TCP Fast Open will be enabled.
Definition: Config.hpp:545
std::uint16_t localPortRange
Number of ports to be tried for outgoing traffic.
Definition: Config.hpp:340
std::uint64_t timeOutRequest
The maximum amount of time a request is allowed to take, in seconds.
Definition: Config.hpp:597
constexpr std::uint16_t httpVersion11
Use HTTP/1.1 only.
Definition: Config.hpp:63
bool cookies
Specifies whether the internal cookie engine will be enabled.
Definition: Config.hpp:155
constexpr std::uint64_t defaultTimeOut
Default connecting time-out, in seconds.
Definition: Config.hpp:99
bool encodingGZip
Specifies whether to request gzip encoding for requested content.
Definition: Config.hpp:271
constexpr std::uint16_t httpVersion1
Use HTTP/1 only.
Definition: Config.hpp:60
std::uint64_t speedLowTime
Number of seconds before a timeout occurs while the transfer speed is below the low speed limit...
Definition: Config.hpp:493
std::string cookiesLoad
The file from which cookies will be read.
Definition: Config.hpp:164
constexpr std::int64_t defaultDnsCacheTimeOut
Default lifetime of DNS cache entries.
Definition: Config.hpp:84
constexpr std::uint16_t defaultConnectionsMax
Default maximum number of connections.
Definition: Config.hpp:81
bool sslVerifyProxyPeer
Specifies whether to verify the authenticity of the proxy's SSL certificate.
Definition: Config.hpp:527
bool verbose
Specifies whether libtidy should produce verbose output.
Definition: Config.hpp:630
bool redirectPost303
Specifies whether to NOT convert POST to GET requests when following 303 redirects.
Definition: Config.hpp:431
void parseBasicOption() override
Parses basic network configuration options.
Definition: Config.hpp:688
bool refererAutomatic
Specifies whether to send an updated HTTP Referer header when automatically redirected.
Definition: Config.hpp:446
void category(const std::string &category)
Sets the category of the subsequent configuration items to be checked for.
Definition: Config.hpp:527
bool redirectPost301
Specifies whether to NOT convert POST to GET requests when following 301 redirects.
Definition: Config.hpp:419
Configuration entries for analyzer threads.
Definition: Config.hpp:132
bool resetTor
Specifies whether to use the TOR control server to request a new identity on connection resets...
Definition: Config.hpp:452
bool sslVerifyStatus
Specifies whether to verify the status of the server's SSL certificate.
Definition: Config.hpp:542
constexpr auto defaultProtocol
Default protocol.
Definition: Config.hpp:105
bool encodingDeflate
Specifies whether to request DEFLATE encoding for requested content.
Definition: Config.hpp:259
std::string proxyTlsSrpPassword
TSL-SRP password for the proxy server used.
Definition: Config.hpp:387
bool proxyTunnelling
Specifies whether to enable proxy tunnelling.
Definition: Config.hpp:399
bool redirectPost302
Specifies whether to NOT convert POST to GET requests when following 302 redirects.
Definition: Config.hpp:425
std::vector< std::string > proxyHeaders
Custom HTTP headers to be sent to the proxy server.
Definition: Config.hpp:373
std::uint64_t tcpKeepAliveIdle
The delay that will be waited before sending keep-alive probes, in seconds.
Definition: Config.hpp:554
bool encodingTransfer
Specifies whether to request HTTP Transfer Encoding.
Definition: Config.hpp:286
std::string userAgent
Custom HTTP User-Agent header to be sent with all HTTP requests.
Definition: Config.hpp:621
std::vector< std::string > dnsResolves
DNS name resolves to be overwritten.
Definition: Config.hpp:229
void resetBase() override
Resets basic network configuration options.
Definition: Config.hpp:771
std::uint64_t tcpKeepAliveInterval
The interval time between keep-alive probes to sent, in seconds.
Definition: Config.hpp:560
std::uint16_t timeOutHappyEyeballs
Number of milliseconds to try to connect only via IPv6 using the Happy Eyeballs algorithm.
Definition: Config.hpp:588
constexpr std::uint16_t httpVersionAny
Use any available HTTP version.
Definition: Config.hpp:57
std::string cookiesSet
Custom HTTP Cookie header independent from the internal cookie engine.
Definition: Config.hpp:197
std::uint16_t connectionsMax
The maximum number of parallel connections.
Definition: Config.hpp:137
std::string proxyTlsSrpUser
TSL-SRP user for the proxy server used.
Definition: Config.hpp:396
bool tcpKeepAlive
Specifies whether TCP keep-alive probing will be enabled.
Definition: Config.hpp:548
std::string dnsInterface
The interface that DNS name resolves should be bound to.
Definition: Config.hpp:215
bool noReUse
Specifies whether to prevent connections from re-using previous ones.
Definition: Config.hpp:348
std::uint64_t timeOut
The maximum amount of time a connection is allowed to take, in seconds.
Definition: Config.hpp:577
std::vector< std::string > dnsServers
DNS servers to be preffered.
Definition: Config.hpp:232
std::uint64_t resetTorOnlyAfter
Number of seconds that need to be parsed before new identity will be requested from the TOR control s...
Definition: Config.hpp:470
constexpr std::uint64_t defaultResetTorOnlyAfter
Default number of seconds that need to have been passed before requesting a new TOR identity...
Definition: Config.hpp:90
std::uint64_t redirectMax
The maximum number of automatic redirects.
Definition: Config.hpp:413
bool encodingBr
Specifies whether to request brotli encoding for requested content.
Definition: Config.hpp:247
std::vector< std::string > cookiesOverwrite
Cookies to be overwritten.
Definition: Config.hpp:171
void parseOption() override=0
Parses additional configuration options.
void warning(const std::string &warning)
Adds a warning to the logging queue.
Definition: Config.hpp:2427
Namespace for networking classes.
Definition: Config.hpp:45
constexpr std::uint64_t defaultTcpKeepAliveInterval
Default interval for TCP Keep-alive probing, in seconds.
Definition: Config.hpp:96
std::string cookiesSave
The file to which cookies will be saved.
Definition: Config.hpp:180
constexpr std::uint64_t defaultTcpKeepAliveIdle
Default delay that will be waited before sending keep-alive probes, in seconds.
Definition: Config.hpp:93