VSI: Use CPLParseMemorySize for CPL_VSIL_CURL_CACHE_SIZE and CPL_VSIL…

…_CURL_CHUNK_SIZE
DelazJ · Oct 24, 2024 · cab4e4f · cab4e4f
1 parent 73f3f38
commit cab4e4f
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 13 deletions.
diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst
@@ -616,7 +616,8 @@ Networking options
       :since: 2.3
 
       Size of global least-recently-used (LRU) cache shared among all downloaded
-      content.
+      content. Value is assumed to represent bytes unless memory units are
+      specified (since GDAL 3.11).
 
 -  .. config:: CPL_VSIL_CURL_USE_HEAD
       :choices: YES, NO
@@ -664,6 +665,9 @@ Networking options
       :choices: <bytes>
       :since: 2.3
 
+      Value is assumed to represent bytes unless memory units are
+      specified (since GDAL 3.11).
+
 -  .. config:: GDAL_INGESTED_BYTES_AT_OPEN
       :since: 2.3
 

diff --git a/port/cpl_string.cpp b/port/cpl_string.cpp
@@ -1848,7 +1848,10 @@ CPLErr CPLParseMemorySize(const char *pszValue, GIntBig *pnValue,
     }
 
     *pnValue = static_cast<GIntBig>(value);
-    *pbUnitSpecified = (unit != nullptr);
+    if (pbUnitSpecified)
+    {
+        *pbUnitSpecified = (unit != nullptr);
+    }
     return CE_None;
 }
 

diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp
@@ -137,31 +137,63 @@ static void VSICURLReadGlobalEnvVariables()
         Initializer()
         {
             constexpr int DOWNLOAD_CHUNK_SIZE_DEFAULT = 16384;
+            const char *pszChunkSize =
+                CPLGetConfigOption("CPL_VSIL_CURL_CHUNK_SIZE", nullptr);
+            GIntBig nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT;
+
+            if (pszChunkSize)
+            {
+                if (CPLParseMemorySize(pszChunkSize, &nChunkSize, nullptr) !=
+                    CE_None)
+                {
+                    CPLError(
+                        CE_Warning, CPLE_AppDefined,
+                        "Could not parse value for CPL_VSIL_CURL_CHUNK_SIZE. "
+                        "Using default value of %d instead.",
+                        DOWNLOAD_CHUNK_SIZE_DEFAULT);
+                }
+            }
 
-            DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = atoi(CPLGetConfigOption(
-                "CPL_VSIL_CURL_CHUNK_SIZE",
-                CPLSPrintf("%d", DOWNLOAD_CHUNK_SIZE_DEFAULT)));
             constexpr int MIN_CHUNK_SIZE = 1024;
             constexpr int MAX_CHUNK_SIZE = 10 * 1024 * 1024;
-            if (DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY < MIN_CHUNK_SIZE ||
-                DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY > MAX_CHUNK_SIZE)
+            if (nChunkSize < MIN_CHUNK_SIZE || nChunkSize > MAX_CHUNK_SIZE)
             {
-                DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY =
-                    DOWNLOAD_CHUNK_SIZE_DEFAULT;
+                nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT;
                 CPLError(CE_Warning, CPLE_AppDefined,
                          "Invalid value for CPL_VSIL_CURL_CHUNK_SIZE. "
                          "Allowed range is [%d, %d]. "
                          "Using CPL_VSIL_CURL_CHUNK_SIZE=%d instead",
                          MIN_CHUNK_SIZE, MAX_CHUNK_SIZE,
-                         DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY);
+                         DOWNLOAD_CHUNK_SIZE_DEFAULT);
             }
+            DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY =
+                static_cast<int>(nChunkSize);
 
             constexpr int N_MAX_REGIONS_DEFAULT = 1000;
             constexpr int CACHE_SIZE_DEFAULT =
                 N_MAX_REGIONS_DEFAULT * DOWNLOAD_CHUNK_SIZE_DEFAULT;
-            GIntBig nCacheSize = CPLAtoGIntBig(
-                CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE",
-                                   CPLSPrintf("%d", CACHE_SIZE_DEFAULT)));
+
+            const char *pszCacheSize =
+                CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE", nullptr);
+            GIntBig nCacheSize = CACHE_SIZE_DEFAULT;
+
+            if (pszCacheSize)
+            {
+                if (CPLParseMemorySize(pszCacheSize, &nCacheSize, nullptr) !=
+                    CE_None)
+                {
+                    CPLError(
+                        CE_Warning, CPLE_AppDefined,
+                        "Could not parse value for CPL_VSIL_CURL_CACHE_SIZE. "
+                        "Using default value of " CPL_FRMT_GIB " instead.",
+                        nCacheSize);
+                }
+            }
+            else
+            {
+                nCacheSize = CACHE_SIZE_DEFAULT;
+            }
+
             const auto nMaxRAM = CPLGetUsablePhysicalRAM();
             const auto nMinVal = DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY;
             auto nMaxVal = static_cast<GIntBig>(INT_MAX) *