Support S3 storage for tenants as AppConfig extension. (awslabs#463)

* Support S3 storage for tenants as AppConfig extension.

This commit adds support for configuring S3 support as part of your SaaS
Boost application. If configured a single S3 bucket will be created for
all your tenants to read/write from, only allowable under a specific
prefix passed as an environment variable to the ECS Task.

Related environment variables for use:
- S3_BUCKET: the name of the s3 bucket created
- S3_ROOT_PREFIX: the prefix under which this tenant is allowed to
  read/write files

* Address PR comments

Co-authored-by: PoeppingT <[email protected]>

client/web/src/components/FormComponents/SaasBoostTooltippedLabel.js

@@ -20,10 +20,20 @@ import { Label, Tooltip } from 'reactstrap'
 export const SaasBoostTooltippedLabel = ({ field, label, tooltip, id, ...props }) => {
   const [tooltipOpen, setTooltipOpen] = useState(false)
   const toggle = () => setTooltipOpen(!tooltipOpen)
-  const tooltipId = id ?? `${field.name}-tooltiptarget`
+  // for some reason it looks like [] are invalid characters in the id selector, 
+  // causing a tooltipId like `services[0].provisionObjectStorage-tooltiptarget` to
+  // be invalid, while something like `services0.provisionObjectStorage-tooltiptarget`
+  // is valid. so we remove invalid characters from the field name before using it
+  // as a tooltipId. also, `.` is considered a query string parameter in the target, and
+  // since we're pulling field names we shouldn't be using it
+  let fieldName = field.name.replaceAll('[', '').replaceAll(']', '').replaceAll('.', '')
+  const tooltipId = id ?? `${fieldName}-tooltiptarget`
 
   return tooltip && label ? (
     <>
+      <Label htmlFor={field.name} id={tooltipId} style={{ borderBottom: '1px dotted black' }}>
+        {label}
+      </Label>
       <Tooltip
         placement="top"
         isOpen={tooltipOpen}
@@ -33,9 +43,6 @@ export const SaasBoostTooltippedLabel = ({ field, label, tooltip, id, ...props }
       >
         {tooltip}
       </Tooltip>
-      <Label htmlFor={field.name} id={tooltipId} style={{ borderBottom: '1px dotted black' }}>
-        {label}
-      </Label>
     </>
   ) : (
     <Label htmlFor={field.name}>{label}</Label>

client/web/src/settings/ApplicationComponent.js

@@ -168,6 +168,7 @@ export function ApplicationComponent(props) {
       operatingSystem: os,
       database: db,
       provisionDb: !!thisService?.database,
+      provisionObjectStorage: !!thisService?.s3,
       windowsVersion: windowsVersion,
       tiers: initialTierValues,
       tombstone: false,
@@ -340,6 +341,7 @@ export function ApplicationComponent(props) {
           }),
           otherwise: Yup.object(),
         }),
+        provisionObjectStorage: Yup.boolean(),
         tiers: Yup.object().when(['tombstone'], (tombstone) => {
           return allTiersValidationSpec(tombstone)
         }),

client/web/src/settings/ApplicationContainer.js

@@ -202,6 +202,7 @@ export function ApplicationContainer(props) {
           operatingSystem,
           ecsLaunchType,
           provisionDb,
+          provisionObjectStorage,
           tombstone,
           database,
           ...rest
@@ -229,6 +230,7 @@ export function ApplicationContainer(props) {
           operatingSystem: operatingSystem === LINUX ? LINUX : windowsVersion,
           ecsLaunchType: (!!ecsLaunchType) ? ecsLaunchType : (operatingSystem === LINUX ? "FARGATE" : "EC2"),
           database: provisionDb ? cleanedDb : null,
+          s3: provisionObjectStorage ? {} : null,
           tiers: cleanedTiersMap,
         }
       }

client/web/src/settings/ServiceSettingsSubform.js

@@ -22,6 +22,7 @@ import { PropTypes } from 'prop-types'
 import { cibWindows, cibLinux } from '@coreui/icons'
 import CIcon from '@coreui/icons-react'
 import DatabaseSubform from './DatabaseSubform'
+import ObjectStoreSubform from './components/ObjectStoreSubform'
 
 const ServiceSettingsSubform = (props) => {
   const { formikErrors, serviceValues, osOptions, dbOptions, serviceName, onFileSelected, isLocked, serviceIndex } = props
@@ -221,6 +222,10 @@ const ServiceSettingsSubform = (props) => {
                     onFileSelected={(file) => onFileSelected(serviceName, file)}
                     setFieldValue={props.setFieldValue}
                   ></DatabaseSubform>
+                  <ObjectStoreSubform
+                    isLocked={isLocked}
+                    formikServicePrefix={'services[' + serviceIndex + ']'}
+                  ></ObjectStoreSubform>
                 </Col>
               </Row>
             </CardBody>

client/web/src/settings/components/ObjectStoreSubform.js

@@ -0,0 +1,59 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PropTypes } from 'prop-types'
+import React, { Fragment } from 'react'
+import {
+  Row,
+  Col,
+  Card,
+  CardBody,
+  CardHeader,
+} from 'reactstrap'
+import {
+  SaasBoostCheckbox,
+} from '../../components/FormComponents'
+import 'rc-slider/assets/index.css'
+
+
+export default class ObjectStoreSubform extends React.Component {
+  render() {
+    return (
+      <Fragment>
+        <Row className="mt-3">
+          <Col xs={12}>
+            <Card>
+              <CardHeader>Object Storage</CardHeader>
+              <CardBody>
+                <SaasBoostCheckbox
+                  id={this.props.formikServicePrefix + '.provisionObjectStorage'}
+                  name={this.props.formikServicePrefix + '.provisionObjectStorage'}
+                  disabled={this.props.isLocked}
+                  label="Provision an S3 bucket for the application."
+                  tooltip="If selected, an S3 bucket will be created on submission and provided as environment variables to the application."
+                />
+              </CardBody>
+            </Card>
+          </Col>
+        </Row>
+      </Fragment>
+    )
+  }
+}
+
+ObjectStoreSubform.propTypes = {
+  isLocked: PropTypes.bool,
+  formikServicePrefix: PropTypes.string,
+}

installer/src/test/java/com/amazon/aws/partners/saasfactory/saasboost/workflow/UpdateWorkflowTest.java

@@ -146,7 +146,7 @@ public void testUpdateActionsFromPaths_customResourcesPath() {
         Set<UpdateAction> expectedActions = EnumSet.of(UpdateAction.CUSTOM_RESOURCES, UpdateAction.RESOURCES);
         List<Path> changedPaths = List.of(
             Path.of("resources/saas-boost.yaml"),
-            Path.of("resources/custom-resources/app-services-ecr-macro/pom.xml"));
+            Path.of("resources/custom-resources/app-services-macro/pom.xml"));
         Collection<UpdateAction> actualActions = updateWorkflow.getUpdateActionsFromPaths(changedPaths);
         assertEquals(expectedActions, actualActions);
         actualActions.forEach(action -> {
@@ -156,7 +156,7 @@ public void testUpdateActionsFromPaths_customResourcesPath() {
             }
             if (action == UpdateAction.CUSTOM_RESOURCES) {
                 assertEquals(1, action.getTargets().size());
-                assertTrue(action.getTargets().contains("app-services-ecr-macro"));
+                assertTrue(action.getTargets().contains("app-services-macro"));
             }
         });
     }

resources/custom-resources/app-services-ecr-macro/pom.xml → resources/custom-resources/app-services-macro/pom.xml

@@ -22,7 +22,7 @@ limitations under the License.
         <artifactId>saasboost-custom-resources</artifactId>
         <version>1.0.0</version>
     </parent>
-    <artifactId>ApplicationServicesEcrMacro</artifactId>
+    <artifactId>ApplicationServicesMacro</artifactId>
     <version>1.0.0</version>
     <packaging>jar</packaging>
     <licenses>

resources/custom-resources/app-services-ecr-macro/src/main/java/com/amazon/aws/partners/saasfactory/saasboost/ApplicationServicesEcrMacro.java → resources/custom-resources/app-services-macro/src/main/java/com/amazon/aws/partners/saasfactory/saasboost/ApplicationServicesMacro.java

@@ -23,9 +23,9 @@
 
 import java.util.*;
 
-public class ApplicationServicesEcrMacro implements RequestHandler<Map<String, Object>, Map<String, Object>> {
+public class ApplicationServicesMacro implements RequestHandler<Map<String, Object>, Map<String, Object>> {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationServicesEcrMacro.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationServicesMacro.class);
     private static final String FRAGMENT = "fragment";
     private static final String REQUEST_ID = "requestId";
     private static final String TEMPLATE_PARAMETERS = "templateParameterValues";
@@ -35,9 +35,12 @@ public class ApplicationServicesEcrMacro implements RequestHandler<Map<String, O
     private static final String ERROR_MSG = "errorMessage";
 
     /**
-     * CloudFormation macro to create a lists of ECR repository resources based on the length of the comma separated
-     * list of application services passed as a template parameter. Returns failure if either "ApplicationServicse"
-     * template parameter is missing.
+     * CloudFormation macro to create resources based on SaaS Boost appConfig objects:
+     *  1/ ECR repository resources for each application service passed as a template parameter. 
+     *     Returns failure if the "ApplicationServices" template parameter is missing.
+     *  2/ S3 bucket resource for all application services if enabled via a template parameter.
+     *     Assumes a missing `AppExtension` parameter means S3 support is not enabled.
+     * 
      * @param event Lambda event containing the CloudFormation request id, fragment, and template parameters
      * @param context Lambda execution context
      * @return CloudFormation macro response of success or failure and the modified template fragment
@@ -52,8 +55,151 @@ public Map<String, Object> handleRequest(Map<String, Object> event, Context cont
         response.put(STATUS, FAILURE);
 
         Map<String, Object> templateParameters = (Map<String, Object>) event.get(TEMPLATE_PARAMETERS);
+        Map<String, Object> template = (Map<String, Object>) event.get(FRAGMENT);
+
+        String ecrError = updateTemplateForEcr(templateParameters, template);
+        if (ecrError != null) {
+            LOGGER.error("Encountered error updating template for ECR repositories: {}");
+            response.put(ERROR_MSG, ecrError);
+            return response;
+        }
+        LOGGER.info("Successfully altered template for ECR repositories");
+
+        String extensionsError = updateTemplateForPooledExtensions(templateParameters, template);
+        if (extensionsError != null) {
+            LOGGER.error("Encountered error updating template for pooled extensions: {}");
+            response.put(ERROR_MSG, extensionsError);
+            return response;
+        }
+        LOGGER.info("Successfully altered template for extensions");
+
+        response.put(FRAGMENT, template);
+        response.put(STATUS, SUCCESS);
+        return response;
+    }
+
+    protected static String updateTemplateForPooledExtensions(
+            final Map<String, Object> templateParameters,
+            Map<String, Object> template) {
+        Set<String> processedExtensions = new HashSet<String>();
+        if (templateParameters.containsKey("AppExtensions") && template.containsKey("Resources")) {
+            String applicationExtensions = (String) templateParameters.get("AppExtensions");
+            if (Utils.isNotEmpty(applicationExtensions)) {
+                String[] extensions = applicationExtensions.split(",");
+                for (String extension : extensions) {
+                    if (processedExtensions.contains(extension)) {
+                        LOGGER.warn("Skipping duplicate extension {}", extension);
+                        continue;
+                    }
+                    switch (extension) {
+                        case "s3": {
+                            String s3Error = updateTemplateForS3(templateParameters, template);
+                            if (s3Error != null) {
+                                LOGGER.error("Processing S3 extension failed: {}", s3Error);
+                                return s3Error;
+                            }
+                            LOGGER.info("Successfully processed s3 extension");
+                            break;
+                        }
+                        default: {
+                            LOGGER.warn("Skipping unknown extension {}", extension);
+                        }
+                    }
+                    processedExtensions.add(extension);
+                }
+            } else {
+                LOGGER.debug("Empty AppExtensions parameter, skipping updating template for extensions");
+            }
+        } else {
+            LOGGER.error("Invalid template, missing AppExtensions parameter or missing Resources");
+            return "Invalid template, missing AppExtensions parameter or missing Resources";
+        }
+        return null;
+    }
+
+    protected static String updateTemplateForS3(final Map<String, Object> templateParameters,
+            Map<String, Object> template) {
+        if (templateParameters.containsKey("Environment")
+                && templateParameters.containsKey("LoggingBucket")) {
+            String environmentName = (String) templateParameters.get("Environment");
+            // Bucket Resource
+            Map<String, Object> s3Resource = s3Resource(environmentName,
+                    (String) templateParameters.get("LoggingBucket"));
+            ((Map<String, Object>) template.get("Resources")).put("TenantStorage", s3Resource);
+
+            // SSM Parameter to easily pass around bucket name
+            Map<String, Object> bucketParameter = Map.of(
+                    "Type", "AWS::SSM::Parameter",
+                    "Properties", Map.of(
+                            "Name", "/saas-boost/" + environmentName + "/TENANT_STORAGE_BUCKET",
+                            "Type", "String",
+                            "Value", Map.of("Ref", "TenantStorage")
+                )
+            );
+            ((Map<String, Object>) template.get("Resources")).put("TenantStorageParam", bucketParameter);
+
+            // Custom Resource to clear the bucket before we delete it
+            Map<String, Object> clearBucketResource = Map.of(
+                    "Type", "Custom::CustomResource",
+                    "Properties", Map.of(
+                            "ServiceToken", "{{resolve:ssm:/saas-boost/" + environmentName + "/CLEAR_BUCKET_ARN}}",
+                            "Bucket", Map.of("Ref", "TenantStorage")
+                    )
+            );
+            ((Map<String, Object>) template.get("Resources")).put("ClearTenantStorageBucket", clearBucketResource);
+        } else {
+            return "Invalid template, missing parameter Environment or LoggingBucket";
+        }
+        return null;
+    }
+
+    protected static Map<String, Object> s3Resource(String environment, String loggingBucket) {
+        Map<String, Object> resourceProperties = new LinkedHashMap<>();
+
+        // tags
+        resourceProperties.put("Tags", List.of(Map.of(
+                "Key", "SaaS Boost",
+                "Value", environment
+        )));
+
+        // encryptionConfiguration
+        resourceProperties.put("BucketEncryption", Map.of(
+                "ServerSideEncryptionConfiguration", List.of(Map.of(
+                        "BucketKeyEnabled", true,
+                        "ServerSideEncryptionByDefault", Map.of("SSEAlgorithm", "AES256")
+                ))
+        ));
+
+        // loggingConfiguration
+        resourceProperties.put("LoggingConfiguration", Map.of(
+                "DestinationBucketName", loggingBucket,
+                "LogFilePrefix", "s3extension-logs"
+        ));
+
+        // ownershipControls
+        resourceProperties.put("OwnershipControls", Map.of(
+                "Rules", List.of(Map.of("ObjectOwnership", "BucketOwnerEnforced"))
+        ));
+
+        // publicAccessBlockConfiguration
+        resourceProperties.put("PublicAccessBlockConfiguration", Map.of(
+                "BlockPublicAcls", true,
+                "BlockPublicPolicy", true,
+                "IgnorePublicAcls", true,
+                "RestrictPublicBuckets", true
+        ));
+
+        Map<String, Object> resource = new LinkedHashMap<>();
+        resource.put("Type", "AWS::S3::Bucket");
+        resource.put("Properties", resourceProperties);
+
+        return resource;
+    }
+
+    protected static String updateTemplateForEcr(
+            final Map<String, Object> templateParameters,
+            Map<String, Object> template) {
         if (templateParameters.containsKey("ApplicationServices")) {
-            Map<String, Object> template = (Map<String, Object>) event.get(FRAGMENT);
             if (template.containsKey("Resources")) {
                 String servicesList = (String) templateParameters.get("ApplicationServices");
                 if (Utils.isNotEmpty(servicesList)) {
@@ -79,13 +225,12 @@ public Map<String, Object> handleRequest(Map<String, Object> event, Context cont
             } else {
                 LOGGER.warn("CloudFormation template fragment does not have Resources");
             }
-            response.put(FRAGMENT, template);
-            response.put(STATUS, SUCCESS);
         } else {
             LOGGER.error("Invalid template, missing parameter ApplicationServices");
-            response.put(ERROR_MSG, "Invalid template, missing parameter ApplicationServices");
+            return "Invalid template, missing parameter ApplicationServices";
         }
-        return response;
+        // no error message implies success?
+        return null;
     }
 
     protected static String cloudFormationResourceName(String name) {