Skip to content

Commit

Permalink
Saas opensource 0118 (aquasecurity#1098)
Browse files Browse the repository at this point in the history 
* space

* syncing with saas

* synced with saas

* space

* saas to opensource syc
  • Loading branch information
@dipsubha06
dipsubha06 authored Jan 27, 2022
1 parent d7ebfcf commit 3222538
Show file tree
Hide file tree
Showing 10 changed files with 140 additions and 111 deletions.
96 changes: 58 additions & 38 deletions collectors/aws/collector.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1923,7 +1923,7 @@ var postcalls = [
}
];

var rateError = [{message: 'rate'}];
var rateError = {message: 'rate', statusCode: 429};

var apiRetryAttempts = 2;
var apiRetryBackoff = 500;
Expand Down Expand Up @@ -2004,26 +2004,15 @@ var collect = function(AWSConfig, settings, callback) {
};

var isRateError = function(err) {
let isError=false;
for (var e in rateError) {
if (err &&
err.statusCode &&
rateError[e] &&
rateError[e].statusCode &&
rateError[e].statusCode.filter(code => {
return code == err.statusCode;
}).length){
isError=true;
break;
} else if (err &&
rateError[e] &&
rateError[e].message &&
err.message &&
err.message.toLowerCase().indexOf(rateError[e].message.toLowerCase())>-1){
isError=true;
break;
}
let isError = false;

if (err && err.statusCode && rateError && rateError.statusCode == err.statusCode) {
isError = true;
} else if (err && rateError && rateError.message && err.message &&
err.message.toLowerCase().indexOf(rateError.message.toLowerCase()) > -1) {
isError = true;
}

return isError;
};

Expand Down Expand Up @@ -2120,15 +2109,42 @@ var collect = function(AWSConfig, settings, callback) {
collection[serviceLower][callKey][region].data = dataToAdd;
}

// If a "paginate" property is set, e.g. NextToken
var nextToken = callObj.paginate;
if (settings.paginate && nextToken && data[nextToken]) {
paginating = true;
var paginateProp = callObj.paginateReqProp ? callObj.paginateReqProp : nextToken;
return execute([paginateProp, data[nextToken]]);
}
if (dataToAdd && dataToAdd.length && settings.identifier){
var localEvent = {};
localEvent.collection = {};
localEvent.collection[serviceLower] = {};
localEvent.collection[serviceLower][callKey] = {};
localEvent.collection[serviceLower][callKey][region] = {};
localEvent.collection[serviceLower][callKey][region].data = dataToAdd;

localEvent.identifier = settings.identifier;

localEvent.previousCollection = settings.previousCollection;

settings.integration(localEvent, function() {
if (debugMode) console.log(`Processed Event: ${JSON.stringify(localEvent)}`);

// If a "paginate" property is set, e.g. NextToken
var nextToken = callObj.paginate;
if (settings.paginate && nextToken && data[nextToken]) {
paginating = true;
var paginateProp = callObj.paginateReqProp ? callObj.paginateReqProp : nextToken;
return execute([paginateProp, data[nextToken]]);
}

return regionCb();
});
} else {
// If a "paginate" property is set, e.g. NextToken
var nextToken = callObj.paginate;
if (settings.paginate && nextToken && data[nextToken]) {
paginating = true;
var paginateProp = callObj.paginateReqProp ? callObj.paginateReqProp : nextToken;
return execute([paginateProp, data[nextToken]]);
}

return regionCb();
return regionCb();
}
};

function execute(nextTokens) { // eslint-disable-line no-inner-declarations
Expand All @@ -2149,14 +2165,13 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](localParams, function(err, data) {
if (isRateError(err)) {
return cb(err);
} else {
return cb(err, data);
}
return cb(err, data);
});
}, function(err, data){
executorCb(err, data);
Expand All @@ -2174,14 +2189,13 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](function(err, data) {
if (isRateError(err)) {
return cb(err);
} else {
return cb(err, data);
}
return cb(err, data);
});
}, function(err, data){
executorCb(err, data);
Expand Down Expand Up @@ -2291,6 +2305,9 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](filter, function(err, data) {
Expand Down Expand Up @@ -2333,6 +2350,9 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](filter, function(err, data) {
Expand Down
100 changes: 60 additions & 40 deletions collectors/aws/collector_multipart.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1492,7 +1492,7 @@ var CALLS_CONFIG = {
POSTCALLS_PARTS: 8
};

var rateError = [{message: 'rate'}];
var rateError = {message: 'rate', statusCode: 429};

var apiRetryAttempts = 2;
var apiRetryBackoff = 500;
Expand Down Expand Up @@ -1580,26 +1580,15 @@ var collect = function(AWSConfig, settings, callback) {
};

var isRateError = function(err) {
let isError=false;
for (var e in rateError) {
if (err &&
err.statusCode &&
rateError[e] &&
rateError[e].statusCode &&
rateError[e].statusCode.filter(code => {
return code == err.statusCode;
}).length){
isError=true;
break;
} else if (err &&
rateError[e] &&
rateError[e].message &&
err.message &&
err.message.toLowerCase().indexOf(rateError[e].message.toLowerCase())>-1){
isError=true;
break;
}
let isError = false;

if (err && err.statusCode && rateError.statusCode == err.statusCode){
isError = true;
} else if (err && rateError && rateError.message && err.message &&
err.message.toLowerCase().indexOf(rateError.message.toLowerCase()) > -1){
isError = true;
}

return isError;
};

Expand Down Expand Up @@ -1670,7 +1659,7 @@ var collect = function(AWSConfig, settings, callback) {
LocalAWSConfig.region = region;

if (callObj.override) {
collectors[serviceLower][callKey](LocalAWSConfig, collection, function() {
collectors[serviceLower][callKey](LocalAWSConfig, collection, retries, function() {
if (callObj.rateLimit) {
setTimeout(function() {
regionCb();
Expand Down Expand Up @@ -1700,15 +1689,42 @@ var collect = function(AWSConfig, settings, callback) {
collection[serviceLower][callKey][region].data = dataToAdd;
}

// If a "paginate" property is set, e.g. NextToken
var nextToken = callObj.paginate;
if (settings.paginate && nextToken && data[nextToken]) {
paginating = true;
var paginateProp = callObj.paginateReqProp ? callObj.paginateReqProp : nextToken;
return execute([paginateProp, data[nextToken]]);
}
if (dataToAdd && dataToAdd.length && settings.identifier){
var localEvent = {};
localEvent.collection = {};
localEvent.collection[serviceLower] = {};
localEvent.collection[serviceLower][callKey] = {};
localEvent.collection[serviceLower][callKey][region] = {};
localEvent.collection[serviceLower][callKey][region].data = dataToAdd;

return regionCb();
localEvent.identifier = settings.identifier;

localEvent.previousCollection = settings.previousCollection;

settings.integration(localEvent, function() {
if (debugMode) console.log(`Processed Event: ${JSON.stringify(localEvent)}`);

// If a "paginate" property is set, e.g. NextToken
var nextToken = callObj.paginate;
if (settings.paginate && nextToken && data[nextToken]) {
paginating = true;
var paginateProp = callObj.paginateReqProp ? callObj.paginateReqProp : nextToken;
return execute([paginateProp, data[nextToken]]);
}

return regionCb();
});
} else {
// If a "paginate" property is set, e.g. NextToken
var nextToken = callObj.paginate;
if (settings.paginate && nextToken && data[nextToken]) {
paginating = true;
var paginateProp = callObj.paginateReqProp ? callObj.paginateReqProp : nextToken;
return execute([paginateProp, data[nextToken]]);
}

return regionCb();
}
};

function execute(nextTokens) { // eslint-disable-line no-inner-declarations
Expand All @@ -1729,14 +1745,13 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](localParams, function(err, data) {
if (isRateError(err)) {
return cb(err);
} else {
return cb(err, data);
}
return cb(err, data);
});
}, function(err, data){
executorCb(err, data);
Expand All @@ -1754,14 +1769,13 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](function(err, data) {
if (isRateError(err)) {
return cb(err);
} else {
return cb(err, data);
}
return cb(err, data);
});
}, function(err, data){
executorCb(err, data);
Expand Down Expand Up @@ -1838,7 +1852,7 @@ var collect = function(AWSConfig, settings, callback) {
if (callObj.signatureVersion) LocalAWSConfig.signatureVersion = callObj.signatureVersion;

if (callObj.override) {
collectors[serviceLower][callKey](LocalAWSConfig, collection, function() {
collectors[serviceLower][callKey](LocalAWSConfig, collection, retries, function() {
if (callObj.rateLimit) {
setTimeout(function() {
regionCb();
Expand Down Expand Up @@ -1876,6 +1890,9 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](filter, function(err, data) {
Expand Down Expand Up @@ -1918,6 +1935,9 @@ var collect = function(AWSConfig, settings, callback) {
console.log(`Trying again in: ${retry_seconds/1000} seconds`);
retries.push({seconds: Math.round(retry_seconds/1000)});
return retry_seconds;
},
errorFilter: function(err) {
return isRateError(err);
}
}, function(cb) {
executor[callKey](filter, function(err, data) {
Expand Down
1 change: 1 addition & 0 deletions docs/gcp.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
name: roles/AquaCSPMSecurityAudit
title: Aqua CSPM Security Audit
- includedPermissions:
- cloudasset.assets.listResource
- cloudkms.cryptoKeys.list
- cloudkms.keyRings.list
- cloudsql.instances.list
Expand Down
Loading

0 comments on commit 3222538

Please sign in to comment.