Skip to content

Commit

Permalink
GME: various robsutness fixes: memory leak, crashes, etc...
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.osgeo.org/gdal/trunk@27178 f0d54148-0727-0410-94bb-9a71ac55c965
  • Loading branch information
@rouault
rouault committed Apr 12, 2014
1 parent 9a58f66 commit 7e31ac6
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 52 deletions.
99 changes: 53 additions & 46 deletions gdal/ogr/ogrsf_frmts/gme/ogrgmedatasource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,17 +209,20 @@ int OGRGMEDataSource::Open( const char * pszFilename, int bUpdateIn)
for(int i=0;papszTables && papszTables[i];i++)
{
papoLayers = (OGRLayer**) CPLRealloc(papoLayers, (nLayers + 1) * sizeof(OGRLayer*));
OGRGMELayer *poGMELayer = new OGRGMELayer(this, papszTables[i]);
OGRGMELayer *poGMELayer = new OGRGMELayer(this, papszTables[i]);
poGMELayer->SetBatchPatchSize(iBatchPatchSize);
if (poGMELayer->GetLayerDefn()) {
if (poGMELayer->GetLayerDefn()) {
papoLayers[nLayers ++] = poGMELayer;
}
else {
delete poGMELayer;
}
}
CSLDestroy(papszTables);
if ( nLayers == 0 ) {
if ( nLayers == 0 ) {
CPLError(CE_Failure, CPLE_AppDefined,
"Could not find any tables.");
return FALSE;
return FALSE;
}
CPLDebug("GME", "Found %d layers", nLayers);
return TRUE;
Expand Down Expand Up @@ -386,52 +389,56 @@ CPLHTTPResult * OGRGMEDataSource::MakeRequest(const char *pszRequest,
if (psResult && psResult->pszErrBuf != NULL)
{
CPLDebug( "GME", "MakeRequest Error Message: %s", psResult->pszErrBuf );
CPLDebug( "GME", "error doc:\n%s\n", psResult->pabyData);
CPLDebug( "GME", "error doc:\n%s\n", psResult->pabyData ? psResult->pabyData : "null");
json_object *error_response = OGRGMEParseJSON((const char *) psResult->pabyData);
CPLHTTPDestroyResult(psResult);
psResult = NULL;
json_object *error_doc = json_object_object_get(error_response, "error");
json_object *errors_doc = json_object_object_get(error_doc, "errors");
array_list *errors_array = json_object_get_array(errors_doc);
int nErrors = array_list_length(errors_array);
for (int i = 0; i < nErrors; i++) {
json_object *error_obj = (json_object *)array_list_get_idx(errors_array, i);
const char* reason = OGRGMEGetJSONString(error_obj, "reason");
const char* domain = OGRGMEGetJSONString(error_obj, "domain");
const char* message = OGRGMEGetJSONString(error_obj, "message");
const char* locationType = OGRGMEGetJSONString(error_obj, "locationType");
const char* location = OGRGMEGetJSONString(error_obj, "location");
if ((nRetries < 10) && EQUAL(reason, "rateLimitExceeded")) {
// Sleep nRetries * 1.0s and retry
nRetries ++;
CPLDebug( "GME", "Got a %s (%d) times.", reason, nRetries );
CPLDebug( "GME", "Sleep for %2.2f to try and avoid qps limiting errors.", 1.0 * nRetries );
CPLSleep( 1.0 * nRetries );
psResult = MakeRequest(pszRequest, pszMoreOptions);
if (psResult)
CPLDebug( "GME", "Got a result after %d retries", nRetries );
else
CPLDebug( "GME", "Didn't get a result after %d retries", nRetries );
nRetries = 0;
}
else if (EQUAL(reason, "authError")) {
CPLDebug( "GME", "Failed to GET %s: %s", pszRequest, message );
CPLError( CE_Failure, CPLE_OpenFailed, "GME: %s", message);
}
else if (EQUAL(reason, "backendError")) {
CPLDebug( "GME", "Backend error retrying: GET %s: %s", pszRequest, message );
psResult = MakeRequest(pszRequest, pszMoreOptions);
}
else {
int code = 444;
json_object *code_child = json_object_object_get(error_doc, "code");
if (code_child != NULL )
code = json_object_get_int(code_child);

CPLDebug( "GME", "MakeRequest Error for %s: %s:%d", pszRequest, reason, code);
CPLError( CE_Failure, CPLE_AppDefined, "GME: %s %s %s: %s - %s",
domain, reason, locationType, location, message );
if( error_response != NULL )
{
json_object *error_doc = json_object_object_get(error_response, "error");
json_object *errors_doc = json_object_object_get(error_doc, "errors");
array_list *errors_array = json_object_get_array(errors_doc);
int nErrors = array_list_length(errors_array);
for (int i = 0; i < nErrors; i++) {
json_object *error_obj = (json_object *)array_list_get_idx(errors_array, i);
const char* reason = OGRGMEGetJSONString(error_obj, "reason", "");
const char* domain = OGRGMEGetJSONString(error_obj, "domain", "");
const char* message = OGRGMEGetJSONString(error_obj, "message", "");
const char* locationType = OGRGMEGetJSONString(error_obj, "locationType", "");
const char* location = OGRGMEGetJSONString(error_obj, "location", "");
if ((nRetries < 10) && EQUAL(reason, "rateLimitExceeded")) {
// Sleep nRetries * 1.0s and retry
nRetries ++;
CPLDebug( "GME", "Got a %s (%d) times.", reason, nRetries );
CPLDebug( "GME", "Sleep for %2.2f to try and avoid qps limiting errors.", 1.0 * nRetries );
CPLSleep( 1.0 * nRetries );
psResult = MakeRequest(pszRequest, pszMoreOptions);
if (psResult)
CPLDebug( "GME", "Got a result after %d retries", nRetries );
else
CPLDebug( "GME", "Didn't get a result after %d retries", nRetries );
nRetries = 0;
}
else if (EQUAL(reason, "authError")) {
CPLDebug( "GME", "Failed to GET %s: %s", pszRequest, message );
CPLError( CE_Failure, CPLE_OpenFailed, "GME: %s", message);
}
else if (EQUAL(reason, "backendError")) {
CPLDebug( "GME", "Backend error retrying: GET %s: %s", pszRequest, message );
psResult = MakeRequest(pszRequest, pszMoreOptions);
}
else {
int code = 444;
json_object *code_child = json_object_object_get(error_doc, "code");
if (code_child != NULL )
code = json_object_get_int(code_child);

CPLDebug( "GME", "MakeRequest Error for %s: %s:%d", pszRequest, reason, code);
CPLError( CE_Failure, CPLE_AppDefined, "GME: %s %s %s: %s - %s",
domain, reason, locationType, location, message );
}
}
json_object_put(error_response);
}
return psResult;
}
Expand Down
16 changes: 10 additions & 6 deletions gdal/ogr/ogrsf_frmts/gme/ogrgmelayer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ OGRGMELayer::~OGRGMELayer()
{
SyncToDisk();
ResetReading();
if( poSRS )
poSRS->Release();
if( poFeatureDefn )
poFeatureDefn->Release();
}

/************************************************************************/
Expand Down Expand Up @@ -345,7 +349,7 @@ OGRFeature *OGRGMELayer::GetNextRawFeature()
CPLString gmeId(gx_id);
omnosIdToGMEKey[++m_nFeaturesRead] = gmeId;
poFeature->SetFID(m_nFeaturesRead);
CPLDebug("GME", "Mapping ids: \"%s\" to %lld", gx_id, m_nFeaturesRead);
CPLDebug("GME", "Mapping ids: \"%s\" to %d", gx_id, (int)m_nFeaturesRead);

/* -------------------------------------------------------------------- */
/* Handle geometry. */
Expand Down Expand Up @@ -465,7 +469,7 @@ OGRErr OGRGMELayer::SetIgnoredFields(const char ** papszFields )

OGRErr OGRGMELayer::BatchPatch()
{
CPLDebug("GME", "BatchPatch() - <%ld>", oListOfDeletedFeatures.size() );
CPLDebug("GME", "BatchPatch() - <%d>", (int)oListOfDeletedFeatures.size() );
return BatchRequest("batchPatch", omnpoUpdatedFeatures);
}

Expand All @@ -475,7 +479,7 @@ OGRErr OGRGMELayer::BatchPatch()

OGRErr OGRGMELayer::BatchInsert()
{
CPLDebug("GME", "BatchInsert() - <%ld>", oListOfDeletedFeatures.size() );
CPLDebug("GME", "BatchInsert() - <%d>", (int)oListOfDeletedFeatures.size() );
return BatchRequest("batchInsert", omnpoInsertedFeatures);
}

Expand All @@ -488,7 +492,7 @@ OGRErr OGRGMELayer::BatchDelete()
json_object *pjoBatchDelete = json_object_new_object();
json_object *pjoGxIds = json_object_new_array();
std::vector<long>::const_iterator fit;
CPLDebug("GME", "BatchDelete() - <%ld>", oListOfDeletedFeatures.size() );
CPLDebug("GME", "BatchDelete() - <%d>", (int)oListOfDeletedFeatures.size() );
for ( fit = oListOfDeletedFeatures.begin(); fit != oListOfDeletedFeatures.end(); fit++)
{
long nFID = *fit;
Expand Down Expand Up @@ -533,7 +537,7 @@ OGRErr OGRGMELayer::BatchRequest(const char *pszMethod, std::map<int, OGRFeature
json_object *pjoBatchDoc = json_object_new_object();
json_object *pjoFeatures = json_object_new_array();
std::map<int, OGRFeature *>::const_iterator fit;
CPLDebug("GME", "BatchRequest('%s', <%ld>)", pszMethod, omnpoFeatures.size() );
CPLDebug("GME", "BatchRequest('%s', <%d>)", pszMethod, (int)omnpoFeatures.size() );
for ( fit = omnpoFeatures.begin(); fit != omnpoFeatures.end(); fit++)
{
long nFID = fit->first;
Expand Down Expand Up @@ -621,7 +625,7 @@ OGRErr OGRGMELayer::CreateFeature( OGRFeature *poFeature )
iGxIdField = nGxId;
if(poFeature->IsFieldSet(iGxIdField)) {
osGxId = poFeature->GetFieldAsString(iGxIdField);
CPLDebug("GME", "Feature already has %d gx_id='%s'", poFeature->GetFID(),
CPLDebug("GME", "Feature already has %ld gx_id='%s'", poFeature->GetFID(),
osGxId.c_str());
}
else {
Expand Down

0 comments on commit 7e31ac6

Please sign in to comment.