From dfd1d1aa4a181f0213b4bf6310dd8fda3ad43013 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Tue, 20 Jun 2023 09:28:17 +1000 Subject: [PATCH] Implement #2414 to allow HTTP session timeout(s) tuning via config (#2425) * Implement #2414 to allow HTTP session timeout(s) tuning via config (taken from v2.5.x) --- config | 6 +++++- docs/USAGE.md | 24 +++++++++++++++++++++++- src/config.d | 48 ++++++++++++++++++++++++++++++++++++++++++------ src/main.d | 9 ++++++--- src/onedrive.d | 18 ++++++++++++++---- src/util.d | 18 ++++++++++++++---- 6 files changed, 104 insertions(+), 19 deletions(-) diff --git a/config b/config index 1ebf742d..807180ea 100644 --- a/config +++ b/config @@ -44,7 +44,6 @@ # sync_dir_permissions = "700" # sync_file_permissions = "600" # rate_limit = "131072" -# operation_timeout = "3600" # webhook_enabled = "false" # webhook_public_url = "" # webhook_listening_host = "" @@ -55,3 +54,8 @@ # display_running_config = "false" # read_only_auth_scope = "false" # cleanup_local_files = "false" +# operation_timeout = "3600" +# dns_timeout = "60" +# connect_timeout = "10" +# data_timeout = "600" +# ip_protocol_version = "0" diff --git a/docs/USAGE.md b/docs/USAGE.md index 86b31c3b..a9e831bf 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -34,6 +34,7 @@ Before reading this document, please ensure you are running application version + [monitor_log_frequency](#monitor_log_frequency) + [min_notify_changes](#min_notify_changes) + [operation_timeout](#operation_timeout) + + [ip_protocol_version](#ip_protocol_version) * [Configuring the client for 'single tenant application' use](#configuring-the-client-for-single-tenant-application-use) * [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier) - [Frequently Asked Configuration Questions](#frequently-asked-configuration-questions) @@ -518,7 +519,6 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con # sync_dir_permissions = "700" # sync_file_permissions = "600" # rate_limit = "131072" -# operation_timeout = "3600" # webhook_enabled = "false" # webhook_public_url = "" # webhook_listening_host = "" @@ -529,6 +529,11 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con # display_running_config = "false" # read_only_auth_scope = "false" # cleanup_local_files = "false" +# operation_timeout = "3600" +# dns_timeout = "60" +# connect_timeout = "10" +# data_timeout = "600" +# ip_protocol_version = "0" ``` ### 'config' file configuration examples: @@ -741,6 +746,23 @@ Example: operation_timeout = "3600" ``` +#### ip_protocol_version +By default, the application will use IPv4 and IPv6 to resolve and communicate with Microsoft OneDrive. In some Linux distributions (most notably Ubuntu and those distributions based on Ubuntu) this will cause problems due to how DNS resolution is being performed. + +To configure the application to use a specific IP version, configure the following in your config file: +```text +# operation_timeout = "3600" +# dns_timeout = "60" +# connect_timeout = "10" +# data_timeout = "600" +ip_protocol_version = "1" + +``` +**Note:** +* A value of 0 will mean the client will use IPv4 and IPv6. This is the default. +* A value of 1 will mean the client will use IPv4 only. +* A value of 2 will mean the client will use IPv6 only. + #### Configuring the client for 'single tenant application' use In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'. To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process. diff --git a/src/config.d b/src/config.d index 99a91486..8c9ba2ff 100644 --- a/src/config.d +++ b/src/config.d @@ -43,6 +43,35 @@ final class Config // Default file permission mode public long defaultFilePermissionMode = 600; public int configuredFilePermissionMode; + + // Bring in v2.5.0 config items + + // HTTP Struct items, used for configuring HTTP() + // Curl Timeout Handling + // libcurl dns_cache_timeout timeout + immutable int defaultDnsTimeout = 60; + // Connect timeout for HTTP|HTTPS connections + immutable int defaultConnectTimeout = 10; + // With the following settings we force + // - if there is no data flow for 10min, abort + // - if the download time for one item exceeds 1h, abort + // + // Timeout for activity on connection + // this translates into Curl's CURLOPT_LOW_SPEED_TIME + // which says: + // It contains the time in number seconds that the + // transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT + // for the library to consider it too slow and abort. + immutable int defaultDataTimeout = 600; + // Maximum time any operation is allowed to take + // This includes dns resolution, connecting, data transfer, etc. + immutable int defaultOperationTimeout = 3600; + // Specify how many redirects should be allowed + immutable int defaultMaxRedirects = 5; + // Specify what IP protocol version should be used when communicating with OneDrive + immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only + + this(string confdirOption) { @@ -122,9 +151,6 @@ final class Config longValues["sync_file_permissions"] = defaultFilePermissionMode; // Configure download / upload rate limits longValues["rate_limit"] = 0; - // maximum time an operation is allowed to take - // This includes dns resolution, connecting, data transfer, etc. - longValues["operation_timeout"] = 3600; // To ensure we do not fill up the load disk, how much disk space should be reserved by default longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes // Webhook options @@ -161,6 +187,19 @@ final class Config // - Enabling this option will add function processing times to the console output // - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance boolValues["display_processing_time"] = false; + + // HTTPS & CURL Operation Settings + // - Maximum time an operation is allowed to take + // This includes dns resolution, connecting, data transfer, etc. + longValues["operation_timeout"] = defaultOperationTimeout; + // libcurl dns_cache_timeout timeout + longValues["dns_timeout"] = defaultDnsTimeout; + // Timeout for HTTPS connections + longValues["connect_timeout"] = defaultConnectTimeout; + // Timeout for activity on a HTTPS connection + longValues["data_timeout"] = defaultDataTimeout; + // What IP protocol version should be used when communicating with OneDrive + longValues["ip_protocol_version"] = defaultIpProtocol; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only // EXPAND USERS HOME DIRECTORY // Determine the users home directory. @@ -451,9 +490,6 @@ final class Config "no-remote-delete", "Do not delete local file 'deletes' from OneDrive when using --upload-only", &boolValues["no_remote_delete"], - "operation-timeout", - "Maximum amount of time (in seconds) an operation is allowed to take", - &longValues["operation_timeout"], "print-token", "Print the access token, useful for debugging", &boolValues["print_token"], diff --git a/src/main.d b/src/main.d index 97317284..688cd1d5 100644 --- a/src/main.d +++ b/src/main.d @@ -761,7 +761,10 @@ int main(string[] args) writeln("Config option 'debug_https' = ", cfg.getValueBool("debug_https")); writeln("Config option 'rate_limit' = ", cfg.getValueLong("rate_limit")); writeln("Config option 'operation_timeout' = ", cfg.getValueLong("operation_timeout")); - + writeln("Config option 'dns_timeout' = ", cfg.getValueLong("dns_timeout")); + writeln("Config option 'connect_timeout' = ", cfg.getValueLong("connect_timeout")); + writeln("Config option 'data_timeout' = ", cfg.getValueLong("data_timeout")); + writeln("Config option 'ip_protocol_version' = ", cfg.getValueLong("ip_protocol_version")); // Is sync_list configured ? writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files")); @@ -853,7 +856,7 @@ int main(string[] args) // Test if OneDrive service can be reached, exit if it cant be reached log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service"); - online = testNetwork(); + online = testNetwork(cfg); if (!online) { // Cant initialise the API as we are not online if (!cfg.getValueBool("monitor")) { @@ -885,7 +888,7 @@ int main(string[] args) Thread.sleep(dur!"seconds"(maxBackoffInterval)); } // perform the re-rty - online = testNetwork(); + online = testNetwork(cfg); if (online) { // We are now online log.log("Internet connectivity to Microsoft OneDrive service has been restored"); diff --git a/src/onedrive.d b/src/onedrive.d index 58849c0c..b333d849 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -214,9 +214,9 @@ final class OneDriveApi http = HTTP(); // Curl Timeout Handling // libcurl dns_cache_timeout timeout - http.dnsTimeout = (dur!"seconds"(60)); + http.dnsTimeout = (dur!"seconds"(cfg.getValueLong("dns_timeout"))); // Timeout for HTTPS connections - http.connectTimeout = (dur!"seconds"(10)); + http.connectTimeout = (dur!"seconds"(cfg.getValueLong("connect_timeout"))); // with the following settings we force // - if there is no data flow for 10min, abort // - if the download time for one item exceeds 1h, abort @@ -227,17 +227,27 @@ final class OneDriveApi // It contains the time in number seconds that the // transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT // for the library to consider it too slow and abort. - http.dataTimeout = (dur!"seconds"(600)); + http.dataTimeout = (dur!"seconds"(cfg.getValueLong("data_timeout"))); // maximum time an operation is allowed to take // This includes dns resolution, connecting, data transfer, etc. http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout"))); + // What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6 + http.handle.set(CurlOption.ipresolve,cfg.getValueLong("ip_protocol_version")); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only // Specify how many redirects should be allowed - http.maxRedirects(5); + http.maxRedirects(cfg.defaultMaxRedirects); // Do we enable curl debugging? if (cfg.getValueBool("debug_https")) { http.verbose = true; .debugResponse = true; + + // Output what options we are using so that in the debug log this can be tracked + log.vdebug("http.dnsTimeout = ", cfg.getValueLong("dns_timeout")); + log.vdebug("http.connectTimeout = ", cfg.getValueLong("connect_timeout")); + log.vdebug("http.dataTimeout = ", cfg.getValueLong("data_timeout")); + log.vdebug("http.operationTimeout = ", cfg.getValueLong("operation_timeout")); + log.vdebug("http.CurlOption.ipresolve = ", cfg.getValueLong("ip_protocol_version")); + log.vdebug("http.maxRedirects = ", cfg.defaultMaxRedirects); } // Update clientId if application_id is set in config file diff --git a/src/util.d b/src/util.d index a46f63bb..5313ecc5 100644 --- a/src/util.d +++ b/src/util.d @@ -15,7 +15,9 @@ import std.json; import std.traits; import qxor; import core.stdc.stdlib; -static import log; + +import log; +import config; shared string deviceName; @@ -114,15 +116,23 @@ Regex!char wild2regex(const(char)[] pattern) } // returns true if the network connection is available -bool testNetwork() +bool testNetwork(Config cfg) { // Use low level HTTP struct auto http = HTTP(); http.url = "https://login.microsoftonline.com"; // DNS lookup timeout - http.dnsTimeout = (dur!"seconds"(5)); + http.dnsTimeout = (dur!"seconds"(cfg.getValueLong("dns_timeout"))); // Timeout for connecting - http.connectTimeout = (dur!"seconds"(5)); + http.connectTimeout = (dur!"seconds"(cfg.getValueLong("connect_timeout"))); + // Data Timeout for HTTPS connections + http.dataTimeout = (dur!"seconds"(cfg.getValueLong("data_timeout"))); + // maximum time any operation is allowed to take + // This includes dns resolution, connecting, data transfer, etc. + http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout"))); + // What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6 + http.handle.set(CurlOption.ipresolve,cfg.getValueLong("ip_protocol_version")); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only + // HTTP connection test method http.method = HTTP.Method.head; // Attempt to contact the Microsoft Online Service