Merge branch 'master' into feature/gcp-documentation-update

.github/workflows/scans_ci.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: codespell-project/actions-codespell@master
         with:
           check_filenames: true
-          skip: ./.github/*,.git,./package.json,./node_modules,./tests,./config,*.png,Dockerfile,./scripts,*.spec.js,./plugins/azure/storageaccounts/storageAccountsAADEnabled.js,./plugins/aws/cloudtrail/cloudtrailBucketAccessLogging.js,./helpers/google/index.js,*zip
+          skip: ./.github/*,.git,./package.json,./package-lock.json,./node_modules,./tests,./config,*.png,Dockerfile,./scripts,*.spec.js,./plugins/azure/storageaccounts/storageAccountsAADEnabled.js,./plugins/aws/cloudtrail/cloudtrailBucketAccessLogging.js,./helpers/google/index.js,*zip
           ignore_words_list: iam,\"tRe\",AKS,aks,optin,callInt,callInt
       - run: npm install
 

README.md

@@ -39,7 +39,7 @@ $ docker run -e AWS_ACCESS_KEY_ID=XX -e AWS_SECRET_ACCESS_KEY=YY cloudsploit:0.0
   + [CloudSploit Config File](#cloudsploit-config-file)
   + [Credential Files](#credential-files)
     + [AWS](#aws)
-    + [Azure](#azur)
+    + [Azure](#azure)
     + [GCP](#gcp)
     + [Oracle OCI](#oracle-oci)
   + [Environment Variables](#environment-variables)

collectors/alibaba/ack/describeClusterDetail.js

@@ -0,0 +1,47 @@
+var ROAClient = require('@alicloud/pop-core').ROAClient;
+
+var apiVersion = '2015-12-15';
+var httpMethod = 'GET';
+var uriPathClusterDetail = '/clusters/';
+var body = '{}';
+var headers = {
+    'Content-Type': 'application/json'
+};
+var requestOption = {timeout: 30000};
+
+module.exports = function(AlibabaConfig, collection, region, callback) {
+    let localConfig = { ...AlibabaConfig };
+    localConfig['endpoint'] = `https://cs.${region}.aliyuncs.com`;
+    localConfig['apiVersion'] = apiVersion;
+    var client = new ROAClient(localConfig);
+
+    var clusters = collection.ack.describeClustersV1[region].data;
+    var totalClusters = clusters.length;
+    var completedRequests = 0;
+    var allRequestsCompleted = function() {
+        callback();
+    };
+
+    var executeSingleClusterDetail = function(cluster) {
+        var clusterId = cluster.cluster_id;
+        var uriPath = `${uriPathClusterDetail}${clusterId}`;
+        client.request(httpMethod, uriPath, {}, body, headers, requestOption).then((res) => {             
+            collection.ack.describeClusterDetail[region][clusterId] = {};
+            collection.ack.describeClusterDetail[region][clusterId].data = res;
+            completedRequests++;
+            if (completedRequests === totalClusters) {
+                allRequestsCompleted();
+            }
+        }, (err) => {
+            collection.ack.describeClusterDetail[region][clusterId] = { err: err };
+            completedRequests++;
+            if (completedRequests === totalClusters) {
+                allRequestsCompleted();
+            }
+        });
+    };
+
+    for (var i = 0; i < totalClusters; i++) {
+        executeSingleClusterDetail(clusters[i]); 
+    }
+};

collectors/alibaba/ack/describeClustersV1.js

@@ -7,7 +7,7 @@ var body = '{}';
 var headers = {
     'Content-Type': 'application/json'
 };
-var requestOption = {};
+var requestOption = {timeout: 30000};
 var pageNumber = 1;
 
 module.exports = function(AlibabaConfig, collection, region, callback) {

collectors/alibaba/collector.js

@@ -21,18 +21,6 @@ var async = require('async');
 var helpers = require(__dirname + '/../../helpers/alibaba');
 var collectors = require(__dirname + '/../../collectors/alibaba');
 
-var regions = helpers.regions();
-
-var regionEndpointMap = {
-    ecs: ['cn-wulanchabu', 'cn-zhangjiak', 'cn-huhehaote', 'cn-heyuan', 'cn-chengdu', 'ap-southeast-2', 'cn-guangzhou',
-        'ap-southeast-3', 'ap-southeast-5', 'ap-northeast-1', 'ap-south-1', 'eu-central-1', 'eu-west-1', 'me-east-1'],
-    kms: regions['kms'],
-    rds: ['cn-zhangjiakou', 'cn-huhehaote', 'cn-chengdu', 'ap-southeast-2', 'ap-southeast-3', 'ap-southeast-5',
-        'ap-northeast-1', 'ap-south-1', 'eu-central-1', 'eu-west-1', 'me-east-1'],
-    actiontrail: regions['actiontrail'],
-    apigateway: regions['apigateway'],
-    tds: ['ap-southeast-3', 'ap-southeast-1']
-};
 
 var globalServices = [
     'OSS',
@@ -171,6 +159,9 @@ var calls = {
         },
         DescribeVersionConfig: {
             apiVersion: '2018-12-03'
+        },
+        DescribeVulConfig: {
+            apiVersion: '2018-12-03'
         }
     }
 };
@@ -324,7 +315,14 @@ var postcalls = [
                 filterValue: ['ApiId', 'GroupId'],
                 apiVersion: '2016-07-14'
             }
-        }
+        },
+        ACK: {
+            describeClusterDetail: {
+                reliesOnService: 'ack',
+                reliesOnCall: 'describeClustersV1',
+                override: true             
+            }
+        },
     }
 ];
 
@@ -333,6 +331,19 @@ var collect = function(AlibabaConfig, settings, callback) {
         return callback(null, calls, postcalls);
     }
 
+    var regions = helpers.regions(settings);
+
+    var regionEndpointMap = {
+        ecs: [ 'cn-hangzhou', 'cn-wulanchabu', 'cn-zhangjiak', 'cn-huhehaote', 'cn-heyuan', 'cn-chengdu', 'ap-southeast-2', 'cn-guangzhou',
+            'ap-southeast-3', 'ap-southeast-5', 'ap-northeast-1', 'ap-south-1', 'eu-central-1', 'eu-west-1', 'me-east-1' ],
+        kms: regions['kms'],
+        rds: [ 'cn-zhangjiakou', 'cn-huhehaote', 'cn-chengdu', 'ap-southeast-2', 'ap-southeast-3', 'ap-southeast-5',
+            'ap-northeast-1', 'ap-south-1', 'eu-central-1', 'eu-west-1', 'me-east-1' ],
+        actiontrail: regions['actiontrail'],
+        apigateway: regions['apigateway'],
+        tds: ['ap-southeast-3', 'ap-southeast-1']
+    };
+
     var collection = {};
 
     async.eachOfLimit(calls, 10, function(call, service, serviceCb) {

collectors/aws/accessanalyzer/listFindings.js

@@ -0,0 +1,49 @@
+var AWS = require('aws-sdk');
+var async = require('async');
+var helpers = require(__dirname + '/../../../helpers/aws');
+
+module.exports = function(AWSConfig, collection, retries, callback) {
+    var accessanalyzer = new AWS.AccessAnalyzer(AWSConfig);
+    async.eachLimit(collection.accessanalyzer.listAnalyzers[AWSConfig.region].data, 15, function(analyzer, cb) {
+        collection.accessanalyzer.listFindings[AWSConfig.region][analyzer.arn] = {};
+        var params = {
+            analyzerArn: analyzer.arn
+        };
+
+        var paginating = false;
+        var paginateCb = function(err, data) {
+            if (err) collection.accessanalyzer.listFindings[AWSConfig.region][analyzer.arn].err = err;
+
+            if (!data) return cb();
+
+            if (paginating && data.findings && data.findings.length &&
+                collection.accessanalyzer.listFindings[AWSConfig.region][analyzer.arn].data.findings &&
+                collection.accessanalyzer.listFindings[AWSConfig.region][analyzer.arn].data.findings.length) {
+                collection.accessanalyzer.listFindings[AWSConfig.region][analyzer.arn].data.findings = collection.accessanalyzer.listFindings[AWSConfig.region][analyzer.arn].data.findings.concat(data.findings);
+            } else {
+                collection.accessanalyzer.listFindings[AWSConfig.region][analyzer.arn].data = data;
+            }
+
+            if (data.nextToken && data.nextToken.length) {
+                paginating = true;
+                return execute(data.nextToken);
+            }
+
+            cb();
+        };
+
+        function execute(nextToken) { // eslint-disable-line no-inner-declarations
+            var localParams = JSON.parse(JSON.stringify(params || {}));
+            if (nextToken) localParams['nextToken'] = nextToken;
+            if (nextToken) {
+                helpers.makeCustomCollectorCall(accessanalyzer, 'listFindings', localParams, retries, null, null, null, paginateCb);
+            } else {
+                helpers.makeCustomCollectorCall(accessanalyzer, 'listFindings', params, retries, null, null, null, paginateCb);
+            }
+        }
+
+        execute();
+    }, function(){
+        callback();
+    });
+};

collectors/aws/apigateway/getIntegration.js

@@ -0,0 +1,52 @@
+var AWS = require('aws-sdk');
+var async = require('async');
+var helpers = require(__dirname + '/../../../helpers/aws');
+
+module.exports = function(AWSConfig, collection, retries, callback) {
+    var apigateway = new AWS.APIGateway(AWSConfig);
+
+    async.eachLimit(collection.apigateway.getRestApis[AWSConfig.region].data, 5, function(api, cb){
+        if (!collection.apigateway.getResources ||
+            !collection.apigateway.getResources[AWSConfig.region] ||
+            !collection.apigateway.getResources[AWSConfig.region][api.id] ||
+            !collection.apigateway.getResources[AWSConfig.region][api.id].data ||
+            !collection.apigateway.getResources[AWSConfig.region][api.id].data.items) {
+            return cb();
+        }
+
+        collection.apigateway.getIntegration[AWSConfig.region][api.id] = {};
+        async.eachLimit(collection.apigateway.getResources[AWSConfig.region][api.id].data.items, 3, function(resource, pCb){
+
+            collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id] = {};
+
+            async.eachOfLimit(resource.resourceMethods, 3, function(methodVal,methodKey, mCb){
+
+                collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id][methodKey] = {};
+
+                let params = {
+                    resourceId: resource.id,
+                    httpMethod: methodKey,
+                    restApiId : api.id,
+                };
+
+                helpers.makeCustomCollectorCall(apigateway, 'getIntegration', params, retries, null, null, null, function(err, data) {
+                    if (err) {
+                        collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id][methodKey].err = err;
+                        return mCb();
+                    }
+
+                    collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id][methodKey].data = data;
+                    mCb();
+                });
+            }, function(){
+                pCb();
+            });
+
+        }, function(){
+            cb();
+        });
+
+    }, function(){
+        callback();
+    });
+};

collectors/aws/appmesh/describeVirtualGateway.js

@@ -0,0 +1,42 @@
+var AWS = require('aws-sdk');
+var async = require('async');
+var helpers = require(__dirname + '/../../../helpers/aws');
+
+module.exports = function(AWSConfig, collection, retries, callback) {
+    var appmesh = new AWS.AppMesh(AWSConfig);
+
+    if (!collection.appmesh ||
+        !collection.appmesh.listMeshes ||
+        !collection.appmesh.listMeshes[AWSConfig.region] ||
+        !collection.appmesh.listMeshes[AWSConfig.region].data) return callback();
+    async.eachLimit(collection.appmesh.listMeshes[AWSConfig.region].data, 5, function(mesh, cb){
+
+        if (!mesh.meshName || !collection.appmesh ||
+            !collection.appmesh.listVirtualGateways ||
+            !collection.appmesh.listVirtualGateways[AWSConfig.region] ||
+            !collection.appmesh.listVirtualGateways[AWSConfig.region][mesh.meshName] ||
+            !collection.appmesh.listVirtualGateways[AWSConfig.region][mesh.meshName].data ||
+            !collection.appmesh.listVirtualGateways[AWSConfig.region][mesh.meshName].data.virtualGateways ||
+            !collection.appmesh.listVirtualGateways[AWSConfig.region][mesh.meshName].data.virtualGateways.length) {
+            return cb();
+        }
+
+        async.eachLimit(collection.appmesh.listVirtualGateways[AWSConfig.region][mesh.meshName].data.virtualGateways, 3, function(gateway, pCb){
+            collection.appmesh.describeVirtualGateway[AWSConfig.region][gateway.virtualGatewayName] = {};
+
+            helpers.makeCustomCollectorCall(appmesh, 'describeVirtualGateway', {virtualGatewayName: gateway.virtualGatewayName,meshName: mesh.meshName}, retries, null, null, null, function(err, data) {
+                if (err) {
+                    collection.appmesh.describeVirtualGateway[AWSConfig.region][gateway.virtualGatewayName].err = err;
+                }
+
+                collection.appmesh.describeVirtualGateway[AWSConfig.region][gateway.virtualGatewayName].data = data;
+                pCb();
+            });
+
+        }, function() {
+            cb();
+        });
+    }, function(){
+        callback();
+    });
+};

collectors/aws/cloudwatch/getEc2MetricStatistics.js

@@ -0,0 +1,38 @@
+var AWS = require('aws-sdk');
+var async = require('async');
+var helpers = require('../../../helpers/aws');
+
+module.exports = function(AWSConfig, collection, retries, callback) {
+    var cloudwatch = new AWS.CloudWatch(AWSConfig);
+
+    async.eachLimit(collection.ec2.describeInstances[AWSConfig.region].data, 10, function(reservation, cb) {
+        reservation.Instances.forEach(instance => {
+            collection.cloudwatch.getEc2MetricStatistics[AWSConfig.region][instance.InstanceId] = {};
+            var endTime = new Date();
+            var startTime = new Date();
+            startTime.setDate(startTime.getDate() - 1);
+            var params = {
+                'MetricName': 'CPUUtilization',
+                'Namespace': 'AWS/EC2',
+                'StartTime': startTime.toISOString(),
+                'EndTime': endTime.toISOString(),
+                'Period': 3600,
+                'Statistics': ['Average'],
+                'Dimensions': [
+                    {
+                        Name: 'InstanceId',
+                        Value: instance.InstanceId
+                    }
+                ]
+            };
+
+            helpers.makeCustomCollectorCall(cloudwatch, 'getMetricStatistics', params,retries, null, null, null, function(err, data) {
+                if (err) collection.cloudwatch.getEc2MetricStatistics[AWSConfig.region][instance.InstanceId].err = err;
+                if (data) collection.cloudwatch.getEc2MetricStatistics[AWSConfig.region][instance.InstanceId].data = data;
+            });
+        });
+        cb();
+    }, function() {
+        callback();
+    });
+};

collectors/aws/cloudwatch/getEsMetricStatistics.js

@@ -5,7 +5,7 @@ var helpers = require(__dirname + '/../../../helpers/aws');
 module.exports = function(AWSConfig, collection, retries, callback) {
     var cloudwatch = new AWS.CloudWatch(AWSConfig);
 
-    async.eachLimit(collection.es.listDomainNames[AWSConfig.region].data, 10, function(domain, cb){        
+    async.eachLimit(collection.opensearch.listDomainNames[AWSConfig.region].data, 10, function(domain, cb){        
         collection.cloudwatch.getEsMetricStatistics[AWSConfig.region][domain.DomainName] = {};
         var endTime = new Date();
         var startTime = new Date();

collectors/aws/cloudwatch/getredshiftMetricStatistics.js

@@ -0,0 +1,39 @@
+var AWS = require('aws-sdk');
+var async = require('async');
+var helpers = require(__dirname + '/../../../helpers/aws');
+
+module.exports = function(AWSConfig, collection, retries, callback) {
+    var cloudwatch = new AWS.CloudWatch(AWSConfig);
+
+    async.eachLimit(collection.redshift.describeClusters[AWSConfig.region].data, 10, function(cluster, cb){        
+        collection.cloudwatch.getredshiftMetricStatistics[AWSConfig.region][cluster.ClusterIdentifier] = {};
+        var endTime = new Date();
+        var startTime = new Date();
+        startTime.setDate(startTime.getDate() - 7);
+        var params = {
+            'MetricName': 'CPUUtilization',
+            'Namespace':'AWS/Redshift',
+            'StartTime': startTime.toISOString(),
+            'EndTime': endTime.toISOString(),
+            'Period': 3600,
+            'Statistics': ['Average'],
+            'Dimensions' : [
+                {
+                    Name: 'ClusterIdentifier',
+                    Value: cluster.ClusterIdentifier
+                }
+            ]
+        };
+
+        helpers.makeCustomCollectorCall(cloudwatch, 'getMetricStatistics', params, retries, null, null, null, function(err, data) {
+            if (err) {
+                collection.cloudwatch.getredshiftMetricStatistics[AWSConfig.region][cluster.ClusterIdentifier].err = err;
+            }
+            collection.cloudwatch.getredshiftMetricStatistics[AWSConfig.region][cluster.ClusterIdentifier].data = data;
+            cb();
+        });
+
+    }, function(){
+        callback();
+    });
+};