Skip to content

Commit

Permalink
Add VSISetPathSpecificOption() / VSIGetPathSpecificOption() / VSIClea…
Browse files Browse the repository at this point in the history 
…rPathSpecificOptions()

Those are just better named functions of the VSISetCredential() /
VSIGetCredential() / VSIClearCredentials() introduced in GDAL 3.5, which
are kept for backward compatibility, but deprecated.
  • Loading branch information
@rouault
rouault committed Oct 6, 2022
1 parent 485bd3b commit fa546d1
Show file tree
Hide file tree
Showing 16 changed files with 464 additions and 168 deletions.
14 changes: 7 additions & 7 deletions autotest/cpp/test_cpl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3619,19 +3619,19 @@ namespace tut
ensure_equals(CPLGetLastErrorType(), CE_None);

{
const char* pszVal = VSIGetCredential("/vsi_test/foo/bar", "FOO", nullptr);
const char* pszVal = VSIGetPathSpecificOption("/vsi_test/foo/bar", "FOO", nullptr);
ensure(pszVal != nullptr);
ensure_equals(std::string(pszVal), std::string("BAR"));
}

{
const char* pszVal = VSIGetCredential("/vsi_test/foo/bar", "FOO2", nullptr);
const char* pszVal = VSIGetPathSpecificOption("/vsi_test/foo/bar", "FOO2", nullptr);
ensure(pszVal != nullptr);
ensure_equals(std::string(pszVal), std::string("BAR2"));
}

{
const char* pszVal = VSIGetCredential("/vsi_test/bar/baz", "BAR", nullptr);
const char* pszVal = VSIGetPathSpecificOption("/vsi_test/bar/baz", "BAR", nullptr);
ensure(pszVal != nullptr);
ensure_equals(std::string(pszVal), std::string("BAZ"));
}
Expand All @@ -3642,11 +3642,11 @@ namespace tut
ensure_equals(std::string(pszVal), std::string("BAR"));
}

VSIClearCredentials("/vsi_test/bar/baz");
VSIClearPathSpecificOptions("/vsi_test/bar/baz");
CPLSetConfigOption("configoptions_FOO", nullptr);

{
const char* pszVal = VSIGetCredential("/vsi_test/bar/baz", "BAR", nullptr);
const char* pszVal = VSIGetPathSpecificOption("/vsi_test/bar/baz", "BAR", nullptr);
ensure(pszVal == nullptr);
}

Expand Down Expand Up @@ -3717,12 +3717,12 @@ namespace tut
ensure_equals(CPLGetLastErrorType(), CE_Warning);

{
const char* pszVal = VSIGetCredential("/vsi_test/foo", "FOO", nullptr);
const char* pszVal = VSIGetPathSpecificOption("/vsi_test/foo", "FOO", nullptr);
ensure(pszVal != nullptr);
}

{
const char* pszVal = VSIGetCredential("/vsi_test/foo", "BAR", nullptr);
const char* pszVal = VSIGetPathSpecificOption("/vsi_test/foo", "BAR", nullptr);
ensure(pszVal == nullptr);
}

Expand Down
46 changes: 23 additions & 23 deletions autotest/gcore/vsicredential.py → autotest/gcore/vsipathspecificoption.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# $Id$
#
# Project: GDAL/OGR Test Suite
# Purpose: Test VSI credentials
# Purpose: Test VSI path specific options
# Author: Even Rouault <even dot rouault at spatialys dot com>
#
###############################################################################
Expand Down Expand Up @@ -33,39 +33,39 @@
from osgeo import gdal


def test_vsicredential():
def test_vsi_path_specific_options():

with pytest.raises(Exception):
assert gdal.GetCredential(None, "key")
assert gdal.GetPathSpecificOption(None, "key")

with pytest.raises(Exception):
assert gdal.GetCredential("prefix", None)
assert gdal.GetPathSpecificOption("prefix", None)

assert gdal.GetCredential("prefix", "key") is None
assert gdal.GetPathSpecificOption("prefix", "key") is None

assert gdal.GetCredential("prefix", "key", "default") == "default"
assert gdal.GetPathSpecificOption("prefix", "key", "default") == "default"

with pytest.raises(Exception):
gdal.SetCredential(None, "key", "value")
gdal.SetPathSpecificOption(None, "key", "value")

with pytest.raises(Exception):
gdal.SetCredential("prefix", None, "value")
gdal.SetPathSpecificOption("prefix", None, "value")

gdal.SetCredential("prefix", "key", "value")
assert gdal.GetCredential("prefix", "key") == "value"
assert gdal.GetCredential("prefix/object", "key") == "value"
assert gdal.GetCredential("prefix", "key", "default") == "value"
assert gdal.GetCredential("another_prefix", "key") is None
gdal.SetPathSpecificOption("prefix", "key", "value")
assert gdal.GetPathSpecificOption("prefix", "key") == "value"
assert gdal.GetPathSpecificOption("prefix/object", "key") == "value"
assert gdal.GetPathSpecificOption("prefix", "key", "default") == "value"
assert gdal.GetPathSpecificOption("another_prefix", "key") is None

gdal.SetCredential("prefix", "key", None)
assert gdal.GetCredential("prefix", "key") is None
gdal.SetPathSpecificOption("prefix", "key", None)
assert gdal.GetPathSpecificOption("prefix", "key") is None

gdal.SetCredential("prefix", "key", "value")
gdal.ClearCredentials("prefix")
assert gdal.GetCredential("prefix", "key") is None
gdal.SetPathSpecificOption("prefix", "key", "value")
gdal.ClearPathSpecificOptions("prefix")
assert gdal.GetPathSpecificOption("prefix", "key") is None

gdal.SetCredential("prefix", "key", "value")
gdal.ClearCredentials("another_prefix")
assert gdal.GetCredential("prefix", "key") == "value"
gdal.ClearCredentials()
assert gdal.GetCredential("prefix", "key") is None
gdal.SetPathSpecificOption("prefix", "key", "value")
gdal.ClearPathSpecificOptions("another_prefix")
assert gdal.GetPathSpecificOption("prefix", "key") == "value"
gdal.ClearPathSpecificOptions()
assert gdal.GetPathSpecificOption("prefix", "key") is None
2 changes: 1 addition & 1 deletion doc/source/user/configoptions.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ the configuration file starts with a ``[directives]`` section that contains a
Starting with GDAL 3.5, a configuration file can also contain credentials
(or more generally options related to a virtual file system) for a given path prefix,
that can also be set with :cpp:func:`VSISetCredential`. Credentials should be put under
that can also be set with :cpp:func:`VSISetPathSpecificOption`. Credentials should be put under
a ``[credentials]`` section, and for each path prefix, under a relative subsection
whose name starts with "[." (e.g. "[.some_arbitrary_name]"), and whose first
key is "path".
Expand Down
4 changes: 2 additions & 2 deletions doc/source/user/virtual_file_systems.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,12 @@ Cloud storage services require setting credentials. For some of them, they can
be provided through configuration files (~/.aws/config, ~/.boto, ..) or through
environment variables / configuration options.

Starting with GDAL 3.5, :cpp:func:`VSISetCredential` can be used to set configuration
Starting with GDAL 3.6, :cpp:func:`VSISetPathSpecificOption` can be used to set configuration
options with a granularity at the level of a file path, which makes it easier if using
the same virtual file system but with different credentials (e.g. different
credentials for bucket "/vsis3/foo" and "/vsis3/bar")

Starting with GDAL 3.5, credentials can be specified in a
Starting with GDAL 3.5, credentials (or path specific options) can be specified in a
:ref:`GDAL configuration file <gdal_configuration_file>`, either in a specific one
explicitly loaded with :cpp:func:`CPLLoadConfigOptionsFromFile`, or
one of the default automatically loaded by :cpp:func:`CPLLoadConfigOptionsFromPredefinedFiles`.
Expand Down
10 changes: 5 additions & 5 deletions port/cpl_alibaba_oss.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,13 +204,13 @@ bool VSIOSSHandleHelper::GetConfiguration(const std::string& osPathForOption,
{
osSecretAccessKey = CSLFetchNameValueDef(papszOptions,
"OSS_SECRET_ACCESS_KEY",
VSIGetCredential(osPathForOption.c_str(), "OSS_SECRET_ACCESS_KEY", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "OSS_SECRET_ACCESS_KEY", ""));

if( !osSecretAccessKey.empty() )
{
osAccessKeyId = CSLFetchNameValueDef(papszOptions,
"OSS_ACCESS_KEY_ID",
VSIGetCredential(osPathForOption.c_str(), "OSS_ACCESS_KEY_ID", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "OSS_ACCESS_KEY_ID", ""));
if( osAccessKeyId.empty() )
{
VSIError(VSIE_AWSInvalidCredentials,
Expand Down Expand Up @@ -248,7 +248,7 @@ VSIOSSHandleHelper* VSIOSSHandleHelper::BuildFromURI( const char* pszURI,

const CPLString osEndpoint = CSLFetchNameValueDef(papszOptions,
"OSS_ENDPOINT",
VSIGetCredential(osPathForOption.c_str(), "OSS_ENDPOINT", "oss-us-east-1.aliyuncs.com"));
VSIGetPathSpecificOption(osPathForOption.c_str(), "OSS_ENDPOINT", "oss-us-east-1.aliyuncs.com"));
CPLString osBucket;
CPLString osObjectKey;
if( pszURI != nullptr && pszURI[0] != '\0' &&
Expand All @@ -257,11 +257,11 @@ VSIOSSHandleHelper* VSIOSSHandleHelper::BuildFromURI( const char* pszURI,
{
return nullptr;
}
const bool bUseHTTPS = CPLTestBool(VSIGetCredential(osPathForOption.c_str(), "OSS_HTTPS", "YES"));
const bool bUseHTTPS = CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(), "OSS_HTTPS", "YES"));
const bool bIsValidNameForVirtualHosting =
osBucket.find('.') == std::string::npos;
const bool bUseVirtualHosting = CPLTestBool(
VSIGetCredential(osPathForOption.c_str(), "OSS_VIRTUAL_HOSTING",
VSIGetPathSpecificOption(osPathForOption.c_str(), "OSS_VIRTUAL_HOSTING",
bIsValidNameForVirtualHosting ? "TRUE" : "FALSE"));
return new VSIOSSHandleHelper(osSecretAccessKey, osAccessKeyId,
osEndpoint,
Expand Down
46 changes: 23 additions & 23 deletions port/cpl_aws.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -773,33 +773,33 @@ bool VSIS3HandleHelper::GetConfigurationFromAssumeRoleWithWebIdentity(bool bForc
}
}

const CPLString roleArn = VSIGetCredential(osPathForOption.c_str(), "AWS_ROLE_ARN", "");
const CPLString roleArn = VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_ROLE_ARN", "");
if( roleArn.empty() )
{
CPLDebug("AWS", "AWS_ROLE_ARN configuration option not defined");
return false;
}

const CPLString webIdentityTokenFile = VSIGetCredential(osPathForOption.c_str(),
const CPLString webIdentityTokenFile = VSIGetPathSpecificOption(osPathForOption.c_str(),
"AWS_WEB_IDENTITY_TOKEN_FILE", "");
if( webIdentityTokenFile.empty() )
{
CPLDebug("AWS", "AWS_WEB_IDENTITY_TOKEN_FILE configuration option not defined");
return false;
}

const CPLString stsRegionalEndpoints = VSIGetCredential(osPathForOption.c_str(),
const CPLString stsRegionalEndpoints = VSIGetPathSpecificOption(osPathForOption.c_str(),
"AWS_STS_REGIONAL_ENDPOINTS", "regional");

std::string osStsDefaultUrl;
if (stsRegionalEndpoints == "regional") {
const CPLString osRegion = VSIGetCredential(osPathForOption.c_str(), "AWS_REGION", "us-east-1");
const CPLString osRegion = VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_REGION", "us-east-1");
osStsDefaultUrl = "https://sts." + osRegion + ".amazonaws.com";
} else {
osStsDefaultUrl = "https://sts.amazonaws.com";
}
const CPLString osStsRootUrl(
VSIGetCredential(osPathForOption.c_str(), "CPL_AWS_STS_ROOT_URL", osStsDefaultUrl.c_str()));
VSIGetPathSpecificOption(osPathForOption.c_str(), "CPL_AWS_STS_ROOT_URL", osStsDefaultUrl.c_str()));

// Get token from web identity token file
CPLString webIdentityToken;
Expand Down Expand Up @@ -887,10 +887,10 @@ bool VSIS3HandleHelper::GetConfigurationFromEC2(bool bForceRefresh,
const CPLString osEC2DefaultURL("http://169.254.169.254");
// coverity[tainted_data]
const CPLString osEC2RootURL(
VSIGetCredential(osPathForOption.c_str(), "CPL_AWS_EC2_API_ROOT_URL", osEC2DefaultURL));
VSIGetPathSpecificOption(osPathForOption.c_str(), "CPL_AWS_EC2_API_ROOT_URL", osEC2DefaultURL));
// coverity[tainted_data]
const CPLString osECSRelativeURI(
VSIGetCredential(osPathForOption.c_str(), "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", ""));
CPLString osToken;
if( osEC2RootURL == osEC2DefaultURL && !osECSRelativeURI.empty() )
{
Expand Down Expand Up @@ -1111,9 +1111,9 @@ bool VSIS3HandleHelper::GetConfigurationFromAWSConfigFiles(
// If AWS_DEFAULT_PROFILE is set (obsolete, no longer documented), use it in priority
// Otherwise use AWS_PROFILE
// Otherwise fallback to "default"
const char* pszProfile = VSIGetCredential(osPathForOption.c_str(), "AWS_DEFAULT_PROFILE", "");
const char* pszProfile = VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_DEFAULT_PROFILE", "");
if( pszProfile[0] == '\0' )
pszProfile = VSIGetCredential(osPathForOption.c_str(), "AWS_PROFILE", "");
pszProfile = VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_PROFILE", "");
const CPLString osProfile(pszProfile[0] != '\0' ? pszProfile : "default");

#ifdef WIN32
Expand All @@ -1133,7 +1133,7 @@ bool VSIS3HandleHelper::GetConfigurationFromAWSConfigFiles(
// GDAL specific config option (mostly for testing purpose, but also
// used in production in some cases)
const char* pszCredentials =
VSIGetCredential(osPathForOption.c_str(), "CPL_AWS_CREDENTIALS_FILE", nullptr );
VSIGetPathSpecificOption(osPathForOption.c_str(), "CPL_AWS_CREDENTIALS_FILE", nullptr );
if( pszCredentials )
{
osCredentials = pszCredentials;
Expand All @@ -1150,7 +1150,7 @@ bool VSIS3HandleHelper::GetConfigurationFromAWSConfigFiles(

// And then ~/.aws/config file (unless AWS_CONFIG_FILE is defined)
const char* pszAWSConfigFileEnv =
VSIGetCredential(osPathForOption.c_str(), "AWS_CONFIG_FILE", nullptr );
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_CONFIG_FILE", nullptr );
CPLString osConfig;
if( pszAWSConfigFileEnv )
{
Expand Down Expand Up @@ -1436,9 +1436,9 @@ bool VSIS3HandleHelper::GetConfiguration(const std::string& osPathForOption,
// AWS_REGION is GDAL specific. Later overloaded by standard
// AWS_DEFAULT_REGION
osRegion = CSLFetchNameValueDef(papszOptions, "AWS_REGION",
VSIGetCredential(osPathForOption.c_str(), "AWS_REGION", "us-east-1"));
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_REGION", "us-east-1"));

if( CPLTestBool(VSIGetCredential(osPathForOption.c_str(), "AWS_NO_SIGN_REQUEST", "NO")) )
if( CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_NO_SIGN_REQUEST", "NO")) )
{
osSecretAccessKey.clear();
osAccessKeyId.clear();
Expand All @@ -1448,12 +1448,12 @@ bool VSIS3HandleHelper::GetConfiguration(const std::string& osPathForOption,

osSecretAccessKey = CSLFetchNameValueDef(papszOptions,
"AWS_SECRET_ACCESS_KEY",
VSIGetCredential(osPathForOption.c_str(), "AWS_SECRET_ACCESS_KEY", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_SECRET_ACCESS_KEY", ""));
if( !osSecretAccessKey.empty() )
{
osAccessKeyId = CSLFetchNameValueDef(papszOptions,
"AWS_ACCESS_KEY_ID",
VSIGetCredential(osPathForOption.c_str(), "AWS_ACCESS_KEY_ID", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_ACCESS_KEY_ID", ""));
if( osAccessKeyId.empty() )
{
VSIError(VSIE_AWSInvalidCredentials,
Expand All @@ -1463,7 +1463,7 @@ bool VSIS3HandleHelper::GetConfiguration(const std::string& osPathForOption,

osSessionToken = CSLFetchNameValueDef(papszOptions,
"AWS_SESSION_TOKEN",
VSIGetCredential(osPathForOption.c_str(), "AWS_SESSION_TOKEN", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_SESSION_TOKEN", ""));
return true;
}

Expand Down Expand Up @@ -1657,16 +1657,16 @@ VSIS3HandleHelper* VSIS3HandleHelper::BuildFromURI( const char* pszURI,
// " This variable overrides the default region of the in-use profile, if set."
const CPLString osDefaultRegion = CSLFetchNameValueDef(
papszOptions, "AWS_DEFAULT_REGION",
VSIGetCredential(osPathForOption.c_str(), "AWS_DEFAULT_REGION", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_DEFAULT_REGION", ""));
if( !osDefaultRegion.empty() )
{
osRegion = osDefaultRegion;
}

const CPLString osEndpoint =
VSIGetCredential(osPathForOption.c_str(), "AWS_S3_ENDPOINT", "s3.amazonaws.com");
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_S3_ENDPOINT", "s3.amazonaws.com");
const CPLString osRequestPayer =
VSIGetCredential(osPathForOption.c_str(), "AWS_REQUEST_PAYER", "");
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_REQUEST_PAYER", "");
CPLString osBucket;
CPLString osObjectKey;
if( pszURI != nullptr && pszURI[0] != '\0' &&
Expand All @@ -1675,12 +1675,12 @@ VSIS3HandleHelper* VSIS3HandleHelper::BuildFromURI( const char* pszURI,
{
return nullptr;
}
const bool bUseHTTPS = CPLTestBool(VSIGetCredential(osPathForOption.c_str(), "AWS_HTTPS", "YES"));
const bool bUseHTTPS = CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_HTTPS", "YES"));
const bool bIsValidNameForVirtualHosting =
osBucket.find('.') == std::string::npos;
const bool bUseVirtualHosting = CPLTestBool(
CSLFetchNameValueDef(papszOptions, "AWS_VIRTUAL_HOSTING",
VSIGetCredential(osPathForOption.c_str(), "AWS_VIRTUAL_HOSTING",
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_VIRTUAL_HOSTING",
bIsValidNameForVirtualHosting ? "TRUE" : "FALSE")));
return new VSIS3HandleHelper(osSecretAccessKey, osAccessKeyId,
osSessionToken,
Expand Down Expand Up @@ -1819,7 +1819,7 @@ VSIS3HandleHelper::GetCurlHeaders( const CPLString& osVerb,

RefreshCredentials(osPathForOption, /* bForceRefresh = */ false);

CPLString osXAMZDate = VSIGetCredential(osPathForOption.c_str(), "AWS_TIMESTAMP", "");
CPLString osXAMZDate = VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_TIMESTAMP", "");
if( osXAMZDate.empty() )
osXAMZDate = CPLGetAWS_SIGN4_Timestamp(time(nullptr));

Expand Down Expand Up @@ -2079,7 +2079,7 @@ CPLString VSIS3HandleHelper::GetSignedURL(CSLConstList papszOptions)
osPathForOption += m_osObjectKey;

CPLString osXAMZDate = CSLFetchNameValueDef(papszOptions, "START_DATE",
VSIGetCredential(osPathForOption.c_str(), "AWS_TIMESTAMP", ""));
VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_TIMESTAMP", ""));
if( osXAMZDate.empty() )
osXAMZDate = CPLGetAWS_SIGN4_Timestamp(time(nullptr));
CPLString osDate(osXAMZDate);
Expand Down
Loading

0 comments on commit fa546d1

Please sign in to comment.