adding region support to cloudsploit

collectors/aws/s3/index.js

@@ -1,10 +1,18 @@
 var AWS = require('aws-sdk');
 var async = require('async');
 var helpers = require(__dirname + '/../../../helpers/aws');
+var awsRegions = require(__dirname + '/../../../helpers/aws/regions.js');
 
 module.exports = function(callKey, forceCloudTrail, AWSConfig, collection, retries, callback) {
     var s3 = new AWS.S3(AWSConfig);
 
+    // Region allow-list (--regions). S3 is collected globally (listBuckets is
+    // account-wide) and each bucket's per-bucket calls are made in the bucket's
+    // own region. When a selection is active we resolve each bucket's region and
+    // skip buckets outside it, so no API call is ever made in an unselected
+    // region. null means "no restriction" (original behavior).
+    var selectedRegions = helpers.getSelectedRegions ? helpers.getSelectedRegions() : null;
+
     var knownBuckets = [];
 
     if (!forceCloudTrail && collection &&
@@ -26,7 +34,7 @@ module.exports = function(callKey, forceCloudTrail, AWSConfig, collection, retri
 
             for (var t in collection.cloudtrail.describeTrails[region].data) {
                 var trail = collection.cloudtrail.describeTrails[region].data[t];
-                
+
                 if (knownBuckets.indexOf(trail.S3BucketName) === -1) {
                     knownBuckets.push(trail.S3BucketName);
                 }
@@ -36,9 +44,10 @@ module.exports = function(callKey, forceCloudTrail, AWSConfig, collection, retri
 
     if (!knownBuckets || !knownBuckets.length) return callback();
 
-    async.eachLimit(knownBuckets, 10, function(bucket, bcb){
-        collection['s3'][callKey][AWSConfig.region][bucket] = {};
-
+    // The original per-bucket call: try against the current (us-east-1) client
+    // and, on a 301, look up the bucket's region and retry there. Used for
+    // buckets whose region is 'global'/undetermined and for unrestricted scans.
+    function makeStandardCall(bucket, bcb) {
         helpers.makeCustomCollectorCall(s3, callKey, {Bucket:bucket}, retries, null, null, null, function(bErr, bData) {
             if (bErr) {
                 collection['s3'][callKey][AWSConfig.region][bucket].err = bErr;
@@ -48,7 +57,7 @@ module.exports = function(callKey, forceCloudTrail, AWSConfig, collection, retri
                         if (locErr || !locData || !locData.LocationConstraint) return bcb();
                         // Special case where location constraint is EU - rewrite as eu-west-1
                         if (locData.LocationConstraint == 'EU') locData.LocationConstraint = 'eu-west-1';
-                        
+
                         var altAWSConfig = JSON.parse(JSON.stringify(AWSConfig));
                         altAWSConfig.region = locData.LocationConstraint;
                         var s3Alt = new AWS.S3(altAWSConfig);
@@ -71,7 +80,88 @@ module.exports = function(callKey, forceCloudTrail, AWSConfig, collection, retri
                 bcb();
             }
         });
+    }
+
+    // Fast path when the bucket's region is already known: call directly in that
+    // region (no 301 probe). Only reached for regions in the selection.
+    function callInRegion(bucket, bucketRegion, bcb) {
+        var target = s3;
+        if (bucketRegion !== AWSConfig.region) {
+            var altAWSConfig = JSON.parse(JSON.stringify(AWSConfig));
+            altAWSConfig.region = bucketRegion;
+            target = new AWS.S3(altAWSConfig);
+        }
+        target[callKey]({Bucket:bucket}, function(err, data){
+            if (err) collection['s3'][callKey][AWSConfig.region][bucket].err = err;
+            else collection['s3'][callKey][AWSConfig.region][bucket].data = data;
+            bcb();
+        });
+    }
+
+    // Map an S3 LocationConstraint to a region id (mirrors getS3BucketLocation).
+    function resolveRegionFromConstraint(constraint) {
+        if (constraint === 'EU') constraint = 'eu-west-1';
+        if (constraint && awsRegions.all.indexOf(constraint) > -1) return constraint;
+        if (constraint) return 'global';
+        return 'us-east-1';
+    }
+
+    // Resolve (and cache) a bucket's region once, shared across every S3
+    // per-bucket call for this scan. getBucketLocation is called against the
+    // us-east-1 client, which returns the constraint without a redirect. The
+    // cache is non-enumerable so it never leaks into the serialized collection.
+    if (!Object.prototype.hasOwnProperty.call(collection.s3, '_bucketRegionCache')) {
+        Object.defineProperty(collection.s3, '_bucketRegionCache', {
+            value: {}, enumerable: false, writable: true, configurable: true
+        });
+    }
+    var locCache = collection.s3._bucketRegionCache;
+
+    function resolveBucketRegion(bucket, cb) {
+        if (Object.prototype.hasOwnProperty.call(locCache, bucket)) return cb(locCache[bucket]);
+
+        // Reuse getBucketLocation data if that call was already gathered.
+        var cached = collection.s3.getBucketLocation &&
+            collection.s3.getBucketLocation[AWSConfig.region] &&
+            collection.s3.getBucketLocation[AWSConfig.region][bucket] &&
+            collection.s3.getBucketLocation[AWSConfig.region][bucket].data;
+        if (cached) {
+            var r = resolveRegionFromConstraint(cached.LocationConstraint);
+            locCache[bucket] = r;
+            return cb(r);
+        }
+
+        helpers.makeCustomCollectorCall(s3, 'getBucketLocation', {Bucket:bucket}, retries, null, null, null, function(locErr, locData) {
+            if (locErr || !locData) { locCache[bucket] = null; return cb(null); }
+            var resolved = resolveRegionFromConstraint(locData.LocationConstraint);
+            locCache[bucket] = resolved;
+            cb(resolved);
+        });
+    }
+
+    async.eachLimit(knownBuckets, 10, function(bucket, bcb){
+        collection['s3'][callKey][AWSConfig.region][bucket] = {};
+
+        // No region allow-list: preserve the original probe-then-redirect flow.
+        if (!selectedRegions || !selectedRegions.length) {
+            return makeStandardCall(bucket, bcb);
+        }
+
+        // Allow-list active: resolve region first and skip out-of-region buckets
+        // so no per-bucket API call hits an unselected region. Buckets that are
+        // 'global' or whose region can't be determined fall back to the original
+        // flow and are always processed.
+        resolveBucketRegion(bucket, function(bucketRegion) {
+            if (bucketRegion && bucketRegion !== 'global') {
+                if (selectedRegions.indexOf(bucketRegion) === -1) {
+                    delete collection['s3'][callKey][AWSConfig.region][bucket];
+                    return bcb();
+                }
+                return callInRegion(bucket, bucketRegion, bcb);
+            }
+            makeStandardCall(bucket, bcb);
+        });
     }, function(){
         callback();
     });
-};
+};

engine.js

@@ -68,6 +68,15 @@ var engine = function(cloudConfig, settings) {
         resourceMap = {};
     }
 
+    // Region restriction (--regions). Register the selected regions at module
+    // scope so collectors/plugins that don't receive `settings` can honor the
+    // allow-list: Azure plugins call helpers.locations() without settings, and
+    // the AWS S3 per-bucket collector needs it to skip out-of-region buckets.
+    if (settings.regions && settings.regions.length) {
+        if (settings.cloud === 'azure') require('./helpers/azure').setRegions(settings.regions);
+        if (settings.cloud === 'aws') require('./helpers/aws').setRegions(settings.regions);
+    }
+
     // Print customization options
     if (settings.compliance) console.log(`INFO: Using compliance modes: ${settings.compliance.join(', ')}`);
     if (settings.govcloud) console.log('INFO: Using AWS GovCloud mode');
@@ -169,7 +178,8 @@ var engine = function(cloudConfig, settings) {
         api_calls: apiCalls,
         paginate: settings.skip_paginate,
         govcloud: settings.govcloud,
-        china: settings.china
+        china: settings.china,
+        regions: settings.regions
     }, function(err, collection) {
         if (err || !collection || !Object.keys(collection).length) return console.log(`ERROR: Unable to obtain API metadata: ${err || 'No data returned'}`);
         outputHandler.writeCollection(collection, settings.cloud);
@@ -197,7 +207,20 @@ var engine = function(cloudConfig, settings) {
                             if (suppressionFilter([key, results[r].region || 'any', results[r].resource || 'any'].join(':'))) {
                                 continue;
                             }
-
+
+                            // When --regions is set, restrict S3 results to the selected regions.
+                            // S3 is a global service (buckets are listed account-wide and always
+                            // collected), but each bucket result is tagged with the bucket's own
+                            // physical region, so buckets outside the selection would otherwise leak
+                            // into the report. Skip those. Results tagged 'global' are always kept.
+                            if (settings.regions && settings.regions.length &&
+                                plugin.category === 'S3' &&
+                                results[r].region &&
+                                results[r].region !== 'global' &&
+                                settings.regions.indexOf(results[r].region) === -1) {
+                                continue;
+                            }
+
                             resultsObject[plugin.title].push(results[r]);
 
                             var complianceMsg = [];

helpers/aws/index.js

@@ -8,16 +8,67 @@ var govRegionsFedRampEast1  = require('./regions_gov_fedramp_east_1.js');
 var govRegionsFedRampWest1  = require('./regions_gov_fedramp_west_1.js');
 var chinaRegions = require('./regions_china.js');
 
+// Meta keys (not per-service scan lists) that must never be narrowed by a
+// region allow-list: `default`/`all` are used for lookups and classification,
+// and `optin` classifies opt-in regions for status/messaging.
+var preserveRegionKeys = ['default', 'all', 'optin'];
+
+// Given the full service -> regions map, return a copy narrowed to only the
+// user-selected regions. Global/single-region services (IAM, S3, CloudFront,
+// Route53, etc.) and the meta keys above are preserved as-is so global checks
+// keep running regardless of which regions were selected.
+var filterRegionsBySelection = function(regionsMap, selectedRegions) {
+    var filtered = {};
+    Object.keys(regionsMap).forEach(function(service) {
+        var serviceRegions = regionsMap[service];
+        if (preserveRegionKeys.indexOf(service) > -1 ||
+            !Array.isArray(serviceRegions) ||
+            serviceRegions.length <= 1) {
+            filtered[service] = serviceRegions;
+            return;
+        }
+        filtered[service] = serviceRegions.filter(function(region) {
+            return selectedRegions.indexOf(region) > -1;
+        });
+    });
+    return filtered;
+};
+
+// User-selected regions (from the --regions flag), stored at module scope so
+// collectors that don't receive `settings` (e.g. the S3 per-bucket collector)
+// can still honor the allow-list. Set once at scan start via setRegions();
+// null means "no restriction".
+var selectedRegions = null;
+
+var setRegions = function(regionList) {
+    selectedRegions = (Array.isArray(regionList) && regionList.length) ? regionList : null;
+};
+
+var getSelectedRegions = function() {
+    return selectedRegions;
+};
+
 var regions = function(settings) {
-    if (settings.govcloud && settings.is_fedramp_type_high && settings.LAMBDA_REGION == 'us-gov-east-1') return govRegionsFedRampEast1;
-    if (settings.govcloud && settings.is_fedramp_type_high && settings.LAMBDA_REGION == 'us-gov-west-1') return govRegionsFedRampWest1;
-    if (settings.govcloud) return govRegions;
-    if (settings.china) return chinaRegions;
-    return regRegions;
+    var regionsMap;
+    if (settings.govcloud && settings.is_fedramp_type_high && settings.LAMBDA_REGION == 'us-gov-east-1') regionsMap = govRegionsFedRampEast1;
+    else if (settings.govcloud && settings.is_fedramp_type_high && settings.LAMBDA_REGION == 'us-gov-west-1') regionsMap = govRegionsFedRampWest1;
+    else if (settings.govcloud) regionsMap = govRegions;
+    else if (settings.china) regionsMap = chinaRegions;
+    else regionsMap = regRegions;
+
+    // Restrict metadata collection and plugin execution to a user-selected set
+    // of regions (from the --regions flag). When unset, behavior is unchanged.
+    if (settings && settings.regions && settings.regions.length) {
+        return filterRegionsBySelection(regionsMap, settings.regions);
+    }
+
+    return regionsMap;
 };
 
 var helpers = {
     regions: regions,
+    setRegions: setRegions,
+    getSelectedRegions: getSelectedRegions,
     MAX_REGIONS_AT_A_TIME: 6,
     CLOUDSPLOIT_EVENTS_BUCKET: 'cloudsploit-engine-trails',
     CLOUDSPLOIT_EVENTS_SNS: 'aqua-cspm-sns-',

index.js

@@ -41,6 +41,12 @@ parser.add_argument('--china', {
     help: 'AWS only. Enables AWS China mode.',
     action: 'store_true'
 });
+parser.add_argument('--regions', {
+    help: 'AWS/Azure only. Restrict the scan to a comma-separated list of regions/locations. ' +
+        'Both metadata collection and plugin execution will only use these regions. ' +
+        'Global services (AWS: IAM, S3, CloudFront, Route53; Azure: AAD, subscriptions, security center, etc.) ' +
+        'always run. E.g. AWS: us-east-1,eu-west-1  Azure: eastus,westeurope'
+});
 parser.add_argument('--csv', { help: 'Output: CSV file' });
 parser.add_argument('--json', { help: 'Output: JSON file' });
 parser.add_argument('--junit', { help: 'Output: Junit file' });
@@ -83,6 +89,18 @@ parser.add_argument('--run-asl', {
 let settings = parser.parse_args();
 let cloudConfig = {};
 
+// Normalize the region allow-list (comma-separated string) into an array so that
+// both the collector and the plugin engine can restrict execution to these regions.
+if (settings.regions && settings.regions.length) {
+    settings.regions = settings.regions
+        .split(',')
+        .map(function(region) { return region.trim(); })
+        .filter(function(region) { return region.length; });
+    if (settings.regions.length) {
+        console.log(`INFO: Restricting scan to regions: ${settings.regions.join(', ')}`);
+    }
+}
+
 // Now execute the scans using the defined configuration information.
 if (!settings.config) {
     settings.cloud = 'aws';