Configuration entries for analyzer threads. More...

#include <Config.hpp>

Network Configuration
std::uint16_t	connectionsMax {defaultConnectionsMax}
	The maximum number of parallel connections. More...

bool	contentLengthIgnore {false}
	Specifies whether the `Content-Length` header in HTTP responses will be ignored. More...

bool	cookies {false}
	Specifies whether the internal cookie engine will be enabled. More...

std::string	cookiesLoad
	The file from which cookies will be read. More...

std::vector< std::string >	cookiesOverwrite
	Cookies to be overwritten. More...

std::string	cookiesSave
	The file to which cookies will be saved. More...

bool	cookiesSession {true}
	Specifies whether to ignore obsolete session cookies. More...

std::string	cookiesSet
	Custom HTTP `Cookie` header independent from the internal cookie engine. More...

std::int64_t	dnsCacheTimeOut {defaultDnsCacheTimeOut}
	The lifetime of DNS cache entries. More...

std::string	dnsDoH
	The URL of a custom DNS-over-HTTPS (DoH) server. More...

std::string	dnsInterface
	The interface that DNS name resolves should be bound to. More...

std::vector< std::string >	dnsResolves
	DNS name resolves to be overwritten. More...

std::vector< std::string >	dnsServers
	DNS servers to be preffered. More...

bool	dnsShuffle {false}
	Specifies whether to shuffle addresses when a host name returns more than one. More...

bool	encodingBr {true}
	Specifies whether to request `brotli` encoding for requested content. More...

bool	encodingDeflate {true}
	Specifies whether to request `DEFLATE` encoding for requested content. More...

bool	encodingGZip {true}
	Specifies whether to request `gzip` encoding for requested content. More...

bool	encodingIdentity {true}
	Specifies whether to (also) request non-compressed encoding for requested content. More...

bool	encodingTransfer {false}
	Specifies whether to request HTTP Transfer Encoding. More...

bool	encodingZstd {false}
	Specifies whether to request `Zstandard` encoding for requested content. More...

std::vector< std::string >	headers
	Custom HTTP headers to be sent with every request. More...

std::vector< std::string >	http200Aliases
	Aliases that will be treated like `HTTP/1.0` `200` `OK`. More...

std::uint16_t	httpVersion {httpVersion2Tls}
	HTTP version(s) to be used. More...

std::string	localInterface
	Interface to be used for outgoing traffic. More...

std::uint16_t	localPort {}
	Port to be used for outgoing traffic. More...

std::uint16_t	localPortRange {1}
	Number of ports to be tried for outgoing traffic. More...

bool	noReUse {false}
	Specifies whether to prevent connections from re-using previous ones. More...

std::string	proxy
	Proxy server used. More...

std::string	proxyAuth
	Authentification for the proxy server used. More...

std::vector< std::string >	proxyHeaders
	Custom HTTP headers to be sent to the proxy server. More...

std::string	proxyPre
	Pre-proxy server to be used. More...

std::string	proxyTlsSrpPassword
	TSL-SRP password for the proxy server used. More...

std::string	proxyTlsSrpUser
	TSL-SRP user for the proxy server used. More...

bool	proxyTunnelling {false}
	Specifies whether to enable proxy tunnelling. More...

bool	redirect {true}
	Specifies whether to follow HTTP `Location` headers for automatic redirects. More...

std::uint64_t	redirectMax {defaultRedirectMax}
	The maximum number of automatic redirects. More...

bool	redirectPost301 {false}
	Specifies whether to NOT convert POST to GET requests when following 301 redirects. More...

bool	redirectPost302 {false}
	Specifies whether to NOT convert POST to GET requests when following 302 redirects. More...

bool	redirectPost303 {false}
	Specifies whether to NOT convert POST to GET requests when following 303 redirects. More...

std::string	referer
	The HTTP `Referer` header to be set. More...

bool	refererAutomatic {false}
	Specifies whether to send an updated HTTP `Referer` header when automatically redirected. More...

bool	resetTor {true}
	Specifies whether to use the TOR control server to request a new identity on connection resets. More...

std::uint64_t	resetTorAfter {}
	Number of seconds until automatically using the TOR control server to request a new identity. More...

std::uint64_t	resetTorOnlyAfter {defaultResetTorOnlyAfter}
	Number of seconds that need to be parsed before new identity will be requested from the TOR control server. More...

std::uint64_t	speedDownLimit {}
	Maximum download speed in bytes per second. More...

std::uint64_t	speedLowLimit {}
	Low speed limit in bytes per second. More...

std::uint64_t	speedLowTime {}
	Number of seconds before a timeout occurs while the transfer speed is below the low speed limit. More...

std::uint64_t	speedUpLimit {}
	Maximum upload speed in bytes per second. More...

bool	sslVerifyHost {true}
	Specifies whether to verify that the SSL certificate is for the server it is known as. More...

bool	sslVerifyPeer {true}
	Specifies whether to verify the authenticity of the server's SSL certificate. More...

bool	sslVerifyProxyHost {true}
	Specifies whether to verify that the SSL certificate is for the proxy server it is known as. More...

bool	sslVerifyProxyPeer {true}
	Specifies whether to verify the authenticity of the proxy's SSL certificate. More...

bool	sslVerifyStatus {false}
	Specifies whether to verify the status of the server's SSL certificate. More...

bool	tcpFastOpen {false}
	Specifies whether TCP Fast Open will be enabled. More...

bool	tcpKeepAlive {false}
	Specifies whether TCP keep-alive probing will be enabled. More...

std::uint64_t	tcpKeepAliveIdle {defaultTcpKeepAliveIdle}
	The delay that will be waited before sending keep-alive probes, in seconds. More...

std::uint64_t	tcpKeepAliveInterval {defaultTcpKeepAliveInterval}
	The interval time between keep-alive probes to sent, in seconds. More...

bool	tcpNagle {false}
	Specifies whether the TCP's Nagle algorithm is enabled on this connection. More...

std::uint64_t	timeOut {defaultTimeOut}
	The maximum amount of time a connection is allowed to take, in seconds. More...

std::uint16_t	timeOutHappyEyeballs {}
	Number of milliseconds to try to connect only via IPv6 using the Happy Eyeballs algorithm. More...

std::uint64_t	timeOutRequest {defaultTimeOutRequest}
	The maximum amount of time a request is allowed to take, in seconds. More...

std::string	tlsSrpUser
	Password used for TLS-SRP authentification. More...

std::string	tlsSrpPassword
	User name used for TLS-SRP authentification. More...

std::string	userAgent
	Custom HTTP `User-Agent` header to be sent with all HTTP requests. More...

bool	verbose {false}
	Specifies whether `libtidy` should produce verbose output. More...

std::string	protocol {defaultProtocol}
	The protocol to be used for HTTP requests. More...

Detailed Description

Configuration entries for analyzer threads.

Warning: Changing the configuration requires updating json/analyzer.json in crawlserv_frontend!

Member Data Documentation

◆ connectionsMax

std::uint16_t crawlservpp::Network::Config::Entries::connectionsMax {defaultConnectionsMax}

The maximum number of parallel connections.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ contentLengthIgnore

bool crawlservpp::Network::Config::Entries::contentLengthIgnore {false}

Specifies whether the Content-Length header in HTTP responses will be ignored.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ cookies

bool crawlservpp::Network::Config::Entries::cookies {false}

Specifies whether the internal cookie engine will be enabled.

The internal cookie engine temporarily stores and re-sends received cookies. They will be reset as soon as the connection is reset.

Use Network::Config::cookiesLoad and Network::Config::cookiesSave for a more permanent solution.

See also: cookiesOverwrite, cookiesSession

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ cookiesLoad

std::string crawlservpp::Network::Config::Entries::cookiesLoad

The file from which cookies will be read.

Will be ignored when empty.

See also: cookies, cookiesOverwrite, cookiesSave, cookiesSession

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ cookiesOverwrite

std::vector<std::string> crawlservpp::Network::Config::Entries::cookiesOverwrite

Cookies to be overwritten.

See also: cookies, cookiesLoad, cookiesSave, cookiesSession

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigCurrent().

◆ cookiesSave

std::string crawlservpp::Network::Config::Entries::cookiesSave

The file to which cookies will be saved.

Will be ignored when empty.

See also: cookies, cookiesLoad, cookiesOverwrite, cookiesSession

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ cookiesSession

bool crawlservpp::Network::Config::Entries::cookiesSession {true}

Specifies whether to ignore obsolete session cookies.

See also: cookies, cookiesLoad, cookiesOverwrite, cookiesSave

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ cookiesSet

std::string crawlservpp::Network::Config::Entries::cookiesSet

Custom HTTP Cookie header independent from the internal cookie engine.

Will be ignored when empty.

Will be overwritten, if a cookie is set during dynamic redirect.

Used by the crawler only.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ dnsCacheTimeOut

std::int64_t crawlservpp::Network::Config::Entries::dnsCacheTimeOut {defaultDnsCacheTimeOut}

The lifetime of DNS cache entries.

Set to -1 for a infinite lifetime.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ dnsDoH

std::string crawlservpp::Network::Config::Entries::dnsDoH

The URL of a custom DNS-over-HTTPS (DoH) server.

Will be ignored when empty.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ dnsInterface

std::string crawlservpp::Network::Config::Entries::dnsInterface

The interface that DNS name resolves should be bound to.

Will be ignored when empty.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ dnsResolves

std::vector<std::string> crawlservpp::Network::Config::Entries::dnsResolves

DNS name resolves to be overwritten.

Each resolve should be in the format HOST:PORT:ADDRESS[,ADDRESS]..., where HOST is the name libcurl will try to resolve, PORT is the port number of the service where @ libcurl wants to connect to the HOST and ADDRESS is one or more numerical IP addresses. Multiple IP addresses need to be separated by comma.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ dnsServers

std::vector<std::string> crawlservpp::Network::Config::Entries::dnsServers

DNS servers to be preffered.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ dnsShuffle

bool crawlservpp::Network::Config::Entries::dnsShuffle {false}

Specifies whether to shuffle addresses when a host name returns more than one.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ encodingBr

bool crawlservpp::Network::Config::Entries::encodingBr {true}

Specifies whether to request brotli encoding for requested content.

If no encoding for the requested content will be specified, no Accept-Encoding header will be sent and no decompression of received content will be performed.

See also: encodingDeflate, encodingGZip, encodingIdentity, encodingZstd

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ encodingDeflate

bool crawlservpp::Network::Config::Entries::encodingDeflate {true}

Specifies whether to request DEFLATE encoding for requested content.

If no encoding for the requested content will be specified, no Accept-Encoding header will be sent and no decompression of received content will be performed.

See also: encodingBr, encodingGZip, encodingIdentity, encodingZstd

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ encodingGZip

bool crawlservpp::Network::Config::Entries::encodingGZip {true}

Specifies whether to request gzip encoding for requested content.

If no encoding for the requested content will be specified, no Accept-Encoding header will be sent and no decompression of received content will be performed.

See also: encodingBr, encodingDeflate, encodingIdentity, encodingZstd

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ encodingIdentity

bool crawlservpp::Network::Config::Entries::encodingIdentity {true}

Specifies whether to (also) request non-compressed encoding for requested content.

If no encoding for the requested content will be specified, no Accept-Encoding header will be sent and no decompression of received content will be performed.

See also: encodingBr, encodingDeflate, encodingGZip, encodingZstd

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ encodingTransfer

bool crawlservpp::Network::Config::Entries::encodingTransfer {false}

Specifies whether to request HTTP Transfer Encoding.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ encodingZstd

bool crawlservpp::Network::Config::Entries::encodingZstd {false}

Specifies whether to request Zstandard encoding for requested content.

If no encoding for the requested content will be specified, no Accept-Encoding header will be sent and no decompression of received content will be performed.

See also: encodingBr, encodingDeflate, encodingGZip, encodingIdentity

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ headers

std::vector<std::string> crawlservpp::Network::Config::Entries::headers

Custom HTTP headers to be sent with every request.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ http200Aliases

std::vector<std::string> crawlservpp::Network::Config::Entries::http200Aliases

Aliases that will be treated like HTTP/1.0 200 OK.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ httpVersion

std::uint16_t crawlservpp::Network::Config::Entries::httpVersion {httpVersion2Tls}

HTTP version(s) to be used.

See also: httpVersionAny, httpVersion1, httpVersion11, httpVersion2, httpVersion2Only, httpVersion2Tls, httpVersion3Only

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ localInterface

std::string crawlservpp::Network::Config::Entries::localInterface

Interface to be used for outgoing traffic.

Will be ignored when empty.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ localPort

std::uint16_t crawlservpp::Network::Config::Entries::localPort {}

Port to be used for outgoing traffic.

Set to zero if any port is fine.

See also: localPortRange

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ localPortRange

std::uint16_t crawlservpp::Network::Config::Entries::localPortRange {1}

Number of ports to be tried for outgoing traffic.

If greater than one, a range of ports starting with localPort will be tried for outgoing traffic.

Set to one or below to not try any additional ports.

See also: localPort

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ noReUse

bool crawlservpp::Network::Config::Entries::noReUse {false}

Specifies whether to prevent connections from re-using previous ones.

This might lead to a large number of re-connects, which might in turn negatively affect performance.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ protocol

std::string crawlservpp::Network::Config::Entries::protocol {defaultProtocol}

The protocol to be used for HTTP requests.

Including :// at the end of the protocol, e.g. https:// or http://.

Referenced by crawlservpp::Network::Config::getProtocol(), crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ proxy

std::string crawlservpp::Network::Config::Entries::proxy

Proxy server used.

Will be ignored when empty.

See also: proxyAuth, proxyHeaders

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ proxyAuth

std::string crawlservpp::Network::Config::Entries::proxyAuth

Authentification for the proxy server used.

The authentification needs to be in the format user:password.

Will be ignored when empty.

See also: proxy, proxyHeaders

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ proxyHeaders

std::vector<std::string> crawlservpp::Network::Config::Entries::proxyHeaders

Custom HTTP headers to be sent to the proxy server.

See also: proxy, proxyAuth

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ proxyPre

std::string crawlservpp::Network::Config::Entries::proxyPre

Pre-proxy server to be used.

Will be ignored when empty.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ proxyTlsSrpPassword

std::string crawlservpp::Network::Config::Entries::proxyTlsSrpPassword

TSL-SRP password for the proxy server used.

Will be ignored when empty.

See also: proxyTlsSrpUser

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ proxyTlsSrpUser

std::string crawlservpp::Network::Config::Entries::proxyTlsSrpUser

TSL-SRP user for the proxy server used.

Will be ignored when empty.

See also: proxyTlsSrpPassword

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ proxyTunnelling

bool crawlservpp::Network::Config::Entries::proxyTunnelling {false}

Specifies whether to enable proxy tunnelling.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ redirect

bool crawlservpp::Network::Config::Entries::redirect {true}

Specifies whether to follow HTTP Location headers for automatic redirects.

See also: redirectMax

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ redirectMax

std::uint64_t crawlservpp::Network::Config::Entries::redirectMax {defaultRedirectMax}

The maximum number of automatic redirects.

Set to -1 to disable the limit.

See also: redirect

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ redirectPost301

bool crawlservpp::Network::Config::Entries::redirectPost301 {false}

Specifies whether to NOT convert POST to GET requests when following 301 redirects.

See also: redirectPost302, redirectPost303

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ redirectPost302

bool crawlservpp::Network::Config::Entries::redirectPost302 {false}

Specifies whether to NOT convert POST to GET requests when following 302 redirects.

See also: redirectPost301, redirectPost303

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ redirectPost303

bool crawlservpp::Network::Config::Entries::redirectPost303 {false}

Specifies whether to NOT convert POST to GET requests when following 303 redirects.

See also: redirectPost301, redirectPost302

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ referer

std::string crawlservpp::Network::Config::Entries::referer

The HTTP Referer header to be set.

No custom HTTP Referer header will be sent if the string is empty.

See also: refererAutomatic

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ refererAutomatic

bool crawlservpp::Network::Config::Entries::refererAutomatic {false}

Specifies whether to send an updated HTTP Referer header when automatically redirected.

See also: referer

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ resetTor

bool crawlservpp::Network::Config::Entries::resetTor {true}

Specifies whether to use the TOR control server to request a new identity on connection resets.

See also: resetTorAfter, resetTorOnlyAfter

Referenced by crawlservpp::Module::Extractor::Thread::onReset(), crawlservpp::Module::Crawler::Thread::onReset(), and crawlservpp::Network::Config::parseBasicOption().

◆ resetTorAfter

std::uint64_t crawlservpp::Network::Config::Entries::resetTorAfter {}

Number of seconds until automatically using the TOR control server to request a new identity.

Set to zero to deactivate the automatic request for a new TOR identity after a specific amount of time has passed.

See also: resetTor, resetTorOnlyAfter

Referenced by crawlservpp::Module::Extractor::Thread::onReset(), crawlservpp::Module::Crawler::Thread::onReset(), and crawlservpp::Network::Config::parseBasicOption().

◆ resetTorOnlyAfter

std::uint64_t crawlservpp::Network::Config::Entries::resetTorOnlyAfter {defaultResetTorOnlyAfter}

Number of seconds that need to be parsed before new identity will be requested from the TOR control server.

See also: resetTor, resetTorAfter

Referenced by crawlservpp::Module::Extractor::Thread::onReset(), crawlservpp::Module::Crawler::Thread::onReset(), and crawlservpp::Network::Config::parseBasicOption().

◆ speedDownLimit

std::uint64_t crawlservpp::Network::Config::Entries::speedDownLimit {}

Maximum download speed in bytes per second.

Set to zero for unlimited download speed.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ speedLowLimit

std::uint64_t crawlservpp::Network::Config::Entries::speedLowLimit {}

Low speed limit in bytes per second.

Set to zero for not using any low speed limit.

See also: speedLowTime

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ speedLowTime

std::uint64_t crawlservpp::Network::Config::Entries::speedLowTime {}

Number of seconds before a timeout occurs while the transfer speed is below the low speed limit.

Set to zero to disable timeouts because of the low speed limit.

See also: speedLowLimit

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ speedUpLimit

std::uint64_t crawlservpp::Network::Config::Entries::speedUpLimit {}

Maximum upload speed in bytes per second.

Set to zero for unlimited upload speed.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ sslVerifyHost

bool crawlservpp::Network::Config::Entries::sslVerifyHost {true}

Specifies whether to verify that the SSL certificate is for the server it is known as.

See also: sslVerifyPeer, sslVerifyProxyHost, sslVerifyProxyPeer, sslVerifyStatus

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ sslVerifyPeer

bool crawlservpp::Network::Config::Entries::sslVerifyPeer {true}

Specifies whether to verify the authenticity of the server's SSL certificate.

See also: sslVerifyHost, sslVerifyProxyHost, sslVerifyProxyPeer, sslVerifyStatus

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ sslVerifyProxyHost

bool crawlservpp::Network::Config::Entries::sslVerifyProxyHost {true}

Specifies whether to verify that the SSL certificate is for the proxy server it is known as.

See also: sslVerifyHost, sslVerifyPeer, sslVerifyProxyPeer, sslVerifyStatus

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ sslVerifyProxyPeer

bool crawlservpp::Network::Config::Entries::sslVerifyProxyPeer {true}

Specifies whether to verify the authenticity of the proxy's SSL certificate.

See also: sslVerifyHost, sslVerifyPeer, sslVerifyProxyHost, sslVerifyStatus

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ sslVerifyStatus

bool crawlservpp::Network::Config::Entries::sslVerifyStatus {false}

Specifies whether to verify the status of the server's SSL certificate.

For this verification, the "Certificate Status Request" TLS extension (i.e. "OCSCP stapling") will be used.

Note: If activated and a server does not support the TLS extension, the verification will fail.

See also: sslVerifyHost, sslVerifyPeer, sslVerifyProxyHost, sslVerifyProxyPeer

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ tcpFastOpen

bool crawlservpp::Network::Config::Entries::tcpFastOpen {false}

Specifies whether TCP Fast Open will be enabled.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ tcpKeepAlive

bool crawlservpp::Network::Config::Entries::tcpKeepAlive {false}

Specifies whether TCP keep-alive probing will be enabled.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ tcpKeepAliveIdle

std::uint64_t crawlservpp::Network::Config::Entries::tcpKeepAliveIdle {defaultTcpKeepAliveIdle}

The delay that will be waited before sending keep-alive probes, in seconds.

See also: tcpKeepAlive, tcpKeepAliveInterval

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ tcpKeepAliveInterval

std::uint64_t crawlservpp::Network::Config::Entries::tcpKeepAliveInterval {defaultTcpKeepAliveInterval}

The interval time between keep-alive probes to sent, in seconds.

See also: tcpKeepAlive, tcpKeepAliveIdle

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ tcpNagle

bool crawlservpp::Network::Config::Entries::tcpNagle {false}

Specifies whether the TCP's Nagle algorithm is enabled on this connection.

The purpose of this algorithm is to try to minimize the number of small packets on the network.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ timeOut

std::uint64_t crawlservpp::Network::Config::Entries::timeOut {defaultTimeOut}

The maximum amount of time a connection is allowed to take, in seconds.

Set to zero to disable timeouts (not recommended).

See also: timeOutRequest

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ timeOutHappyEyeballs

std::uint16_t crawlservpp::Network::Config::Entries::timeOutHappyEyeballs {}

Number of milliseconds to try to connect only via IPv6 using the Happy Eyeballs algorithm.

After this amount of milliseconds, a connection attempt is made to the IPv4 address in parallel.

Set to zero for using the default value provided by libcurl.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ timeOutRequest

std::uint64_t crawlservpp::Network::Config::Entries::timeOutRequest {defaultTimeOutRequest}

The maximum amount of time a request is allowed to take, in seconds.

Set to zero to disable timeouts (not recommended).

See also: timeOut

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ tlsSrpPassword

std::string crawlservpp::Network::Config::Entries::tlsSrpPassword

User name used for TLS-SRP authentification.

Will be ignored when both tlsSrpUser and tlsSrpPassword are empty.

See also: tlsSrpUser

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ tlsSrpUser

std::string crawlservpp::Network::Config::Entries::tlsSrpUser

Password used for TLS-SRP authentification.

Will be ignored when both tlsSrpUser and tlsSrpPassword are empty.

See also: tlsSrpPassword

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ userAgent

std::string crawlservpp::Network::Config::Entries::userAgent

Custom HTTP User-Agent header to be sent with all HTTP requests.

No HTTP User-Agent header will be sent when empty.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

◆ verbose

bool crawlservpp::Network::Config::Entries::verbose {false}

Specifies whether libtidy should produce verbose output.

Warning: Should be used for debugging purposes only!

The output will be written to stdout.

Referenced by crawlservpp::Network::Config::parseBasicOption(), and crawlservpp::Network::Curl::setConfigGlobal().

The documentation for this struct was generated from the following file:

Network/Config.hpp

Network Configuration

Detailed Description

Member Data Documentation

◆ connectionsMax

◆ contentLengthIgnore

◆ cookies

◆ cookiesLoad

◆ cookiesOverwrite

◆ cookiesSave

◆ cookiesSession

◆ cookiesSet

◆ dnsCacheTimeOut

◆ dnsDoH

◆ dnsInterface

◆ dnsResolves

◆ dnsServers

◆ dnsShuffle

◆ encodingBr

◆ encodingDeflate

◆ encodingGZip

◆ encodingIdentity

◆ encodingTransfer

◆ encodingZstd

◆ headers

◆ http200Aliases

◆ httpVersion

◆ localInterface

◆ localPort

◆ localPortRange

◆ noReUse

◆ protocol

◆ proxy

◆ proxyAuth

◆ proxyHeaders

◆ proxyPre

◆ proxyTlsSrpPassword

◆ proxyTlsSrpUser

◆ proxyTunnelling

◆ redirect

◆ redirectMax

◆ redirectPost301

◆ redirectPost302

◆ redirectPost303

◆ referer

◆ refererAutomatic

◆ resetTor

◆ resetTorAfter

◆ resetTorOnlyAfter

◆ speedDownLimit

◆ speedLowLimit

◆ speedLowTime

◆ speedUpLimit

◆ sslVerifyHost

◆ sslVerifyPeer

◆ sslVerifyProxyHost

◆ sslVerifyProxyPeer

◆ sslVerifyStatus

◆ tcpFastOpen

◆ tcpKeepAlive

◆ tcpKeepAliveIdle

◆ tcpKeepAliveInterval

◆ tcpNagle

◆ timeOut

◆ timeOutHappyEyeballs

◆ timeOutRequest

◆ tlsSrpPassword

◆ tlsSrpUser

◆ userAgent

◆ verbose