merge StateVoicesNational#1478 aflcio:scaling-feb20-3-messagej into s…

…tage-main for Spoke version 5.5

.env.example

@@ -27,8 +27,7 @@ SESSION_SECRET=set_this_in_production
 DEFAULT_SERVICE=twilio
 NEXMO_API_KEY=
 NEXMO_API_SECRET=
-TWILIO_API_KEY=
-TWILIO_APPLICATION_SID=
+TWILIO_ACCOUNT_SID=
 TWILIO_AUTH_TOKEN=
 TWILIO_MESSAGE_SERVICE_SID=
 TWILIO_STATUS_CALLBACK_URL=

__test__/e2e/.env.e2e

@@ -24,8 +24,7 @@ SESSION_SECRET=set_this_in_production
 DEFAULT_SERVICE=twilio
 NEXMO_API_KEY=
 NEXMO_API_SECRET=
-TWILIO_API_KEY=
-TWILIO_APPLICATION_SID=
+TWILIO_ACCOUNT_SID=
 TWILIO_AUTH_TOKEN=
 TWILIO_MESSAGE_SERVICE_SID=
 TWILIO_STATUS_CALLBACK_URL=

__test__/lib.test.js

@@ -53,7 +53,7 @@ describe("getConfig/hasConfig", () => {
   });
 
   it("should return false for hasConfig for blank set var", () => {
-    expect(hasConfig("TWILIO_API_KEY")).toBe(false);
+    expect(hasConfig("TWILIO_ACCOUNT_SID")).toBe(false);
   });
 
   it("should return true for hasConfig for set var", () => {

__test__/server/api/lib/crypto.test.js

@@ -0,0 +1,30 @@
+import crypto from "../../../../src/server/api/lib/crypto";
+
+beforeEach(() => {
+  jest.resetModules();
+});
+
+it("encrypted value should be different", () => {
+  const plaintext = "test_auth_token";
+  const encrypted = crypto.symmetricEncrypt(plaintext);
+  expect(encrypted).not.toEqual(plaintext);
+});
+
+it("decrypted value should match original", () => {
+  const plaintext = "another_test_auth_token";
+  const encrypted = crypto.symmetricEncrypt(plaintext);
+  const decrypted = crypto.symmetricDecrypt(encrypted);
+  expect(decrypted).toEqual(plaintext);
+});
+
+it("session secret must exist", () => {
+  function encrypt() {
+    delete global.SESSION_SECRET;
+    const crypto2 = require("../../../../src/server/api/lib/crypto");
+    const plaintext = "foo";
+    const encrypted = crypto2.symmetricEncrypt(plaintext);
+  }
+  expect(encrypt).toThrowError(
+    "The SESSION_SECRET environment variable must be set to use crypto functions!"
+  );
+});

__test__/server/api/lib/twilio.test.js

@@ -14,6 +14,7 @@ import {
   createUser,
   createInvite,
   createOrganization,
+  setTwilioAuth,
   createCampaign,
   saveCampaign,
   copyCampaign,
@@ -28,12 +29,15 @@ import {
 
 let testAdminUser;
 let testInvite;
+let testInvite2;
 let testOrganization;
+let testOrganization2;
 let testCampaign;
 let testTexterUser;
 let testTexterUser2;
 let testContacts;
 let organizationId;
+let organizationId2;
 let assignmentId;
 let dbCampaignContact;
 let queryLog;
@@ -59,6 +63,11 @@ beforeEach(async () => {
   assignmentId = dbCampaignContact.assignment_id;
   await createScript(testAdminUser, testCampaign);
   await startCampaign(testAdminUser, testCampaign);
+  testInvite2 = await createInvite();
+  testOrganization2 = await createOrganization(testAdminUser, testInvite2);
+  organizationId2 = testOrganization2.data.createOrganization.id;
+  await setTwilioAuth(testAdminUser, testOrganization2);
+
   // use caching
   await cacheableData.organization.load(organizationId);
   await cacheableData.campaign.load(testCampaign.id, { forceLoad: true });
@@ -359,6 +368,18 @@ it("handleDeliveryReport error", async () => {
   }
 });
 
+it("orgs should have separate twilio credentials", async () => {
+  const org1 = await cacheableData.organization.load(organizationId);
+  const org1Auth = await cacheableData.organization.getTwilioAuth(org1);
+  expect(org1Auth.authToken).toBeUndefined();
+  expect(org1Auth.accountSid).toBeUndefined();
+
+  const org2 = await cacheableData.organization.load(organizationId2);
+  const org2Auth = await cacheableData.organization.getTwilioAuth(org2);
+  expect(org2Auth.authToken).toBe("test_twlio_auth_token");
+  expect(org2Auth.accountSid).toBe("test_twilio_account_sid");
+});
+
 // FUTURE
 // * parseMessageText
 // * convertMessagePartsToMessage

__test__/test_helpers.js

@@ -160,6 +160,45 @@ export async function createOrganization(user, invite) {
   return await graphql(mySchema, orgQuery, rootValue, context, variables);
 }
 
+export async function setTwilioAuth(user, organization) {
+  const rootValue = {};
+  const accountSid = "test_twilio_account_sid";
+  const authToken = "test_twlio_auth_token";
+  const messageServiceSid = "test_message_service";
+  const orgId = organization.data.createOrganization.id;
+
+  const context = getContext({ user });
+
+  const twilioQuery = `
+      mutation updateTwilioAuth(
+        $twilioAccountSid: String
+        $twilioAuthToken: String
+        $twilioMessageServiceSid: String
+        $organizationId: String!
+      ) {
+        updateTwilioAuth(
+          twilioAccountSid: $twilioAccountSid
+          twilioAuthToken: $twilioAuthToken
+          twilioMessageServiceSid: $twilioMessageServiceSid
+          organizationId: $organizationId
+        ) {
+          id
+          twilioAccountSid
+          twilioAuthToken
+          twilioMessageServiceSid
+        }
+      }`;
+
+  const variables = {
+    organizationId: orgId,
+    twilioAccountSid: accountSid,
+    twilioAuthToken: authToken,
+    twilioMessageServiceSid: messageServiceSid
+  };
+
+  return await graphql(mySchema, twilioQuery, rootValue, context, variables);
+}
+
 export async function createCampaign(
   user,
   organization,

app.json

@@ -49,12 +49,7 @@
       "value": "twilio"
     },
 
-    "TWILIO_API_KEY": {
-      "description": "for twilio integration and connected to twilio account",
-      "required": true
-    },
-
-    "TWILIO_APPLICATION_SID": {
+    "TWILIO_ACCOUNT_SID": {
       "description": "for twilio integration and connected to twilio account",
       "required": true
     },

deploy/lambda-env.json

@@ -3,7 +3,7 @@
   "JOBS_SAME_PROCESS": "1",
   "SUPPRESS_SEED_CALLS": "1",
   "AWS_ACCESS_AVAILABLE": "1",
-  "AWS_S3_BUCKET_NAME":  "spoke.example.com",
+  "AWS_S3_BUCKET_NAME": "spoke.example.com",
   "APOLLO_OPTICS_KEY": "",
   "DEFAULT_SERVICE": "twilio",
   "OUTPUT_DIR": "./build",
@@ -26,9 +26,7 @@
   "SESSION_SECRET": "XXXXXXXXXXXXXXXXXXXXXXXXXX",
   "NEXMO_API_KEY": "",
   "NEXMO_API_SECRET": "",
-  "TWILIO_API_KEY": "XXXXXXXXXXXXXXXXXXXXXXXXXXX",
   "TWILIO_MESSAGE_SERVICE_SID": "XXXXXXXXXXXXXXXXXXXXXXXXX",
-  "TWILIO_APPLICATION_SID": "XXXXXXXXXXXXXXXXXXXXXXXXXXXX",
   "TWILIO_ACCOUNT_SID": "XXXXXXXXXXXXXXXXXXXXXXXXXXX",
   "TWILIO_AUTH_TOKEN": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx",
   "TWILIO_STATUS_CALLBACK_URL": "https://spoke.example.com/twilio-message-report",

deploy/spoke-pm2.config.js.template

@@ -30,8 +30,7 @@ const env_production = {
   SESSION_SECRET:'',
   NEXMO_API_KEY:'',
   NEXMO_API_SECRET:'',
-  TWILIO_API_KEY:'',
-  TWILIO_APPLICATION_SID:'',
+  TWILIO_ACCOUNT_SID:'',
   TWILIO_AUTH_TOKEN:'',
   TWILIO_STATUS_CALLBACK_URL:'http://example.com:3000/twilio',
   PHONE_NUMBER_COUNTRY: 'US',

dev-tools/symmetric-decrypt.js

@@ -0,0 +1,10 @@
+// Run this script from the top level directory to decrypt a value:
+//
+//    node ./dev-tools/symmetric-decrypt.js ValueToBeDecrypted
+
+require("dotenv").config();
+const { symmetricDecrypt } = require("../src/server/api/lib/crypto");
+
+const result = symmetricDecrypt(process.argv[2]);
+console.log(result);
+process.exit(0);

dev-tools/symmetric-encrypt.js

@@ -0,0 +1,10 @@
+// Run this script from the top level directory to encrypt a value:
+//
+//    node ./dev-tools/symmetric-encrypt.js ValueToBeEncrypted
+
+require("dotenv").config();
+const { symmetricEncrypt } = require("../src/server/api/lib/crypto");
+
+const result = symmetricEncrypt(process.argv[2]);
+console.log(result);
+process.exit(0);

docs/HOWTO-configure-auth0.md

@@ -4,18 +4,17 @@ There are two authentication backends supported for Spoke.  The first and defaul
 [https://auth0.com](https://auth0.com). This service allows support for e.g. Google authentication
 and others, and supports things like password resets and account management separate from Spoke.
 
-The alternative and default for development environments is local login, where passwords are hashed locally
-in the database and resets, etc are administered all within Spoke.  While good for development, we
-believe Auth0 still provides better security for production environments.  Below are the steps to configure
-Spoke for Auth0.
+The alternative and default for development environments is local login, where passwords are hashed locally in the database and resets, etc are administered all within Spoke.  While good for development, we
+believe Auth0 still provides better security for production environments.  Below are the steps to configure Spoke for Auth0.
 
 ## Configuration Steps
 
-1. First configure the environment variable `PASSPORT_STRATEGY=auth0` in `.env` or wherever to configure Spoke environment
-variables.
+Note for users following [Instructions for One-Click Deployment to Heroku](https://github.com/MoveOnOrg/Spoke/blob/main/docs/HOWTO_HEROKU_DEPLOY.md): The fields to edit are listed alphabetically under the "Config Vars" heading.
+
+1. First configure the environment variable `PASSPORT_STRATEGY=auth0` in `.env` or wherever you configure Spoke environment variables.
 2. Create an [Auth0](https://auth0.com) account. In your Auth0 account, go to [Applications](https://manage.auth0.com/#/applications/), click on `Default App` and then grab your Client ID, Client Secret, and your Auth0 domain (should look like xxx.auth0.com). Add those inside your `.env` file (AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET, AUTH0_DOMAIN respectively).
 3. Run `yarn dev` to create and populate the tables.
-4. In your Auth0 app settings, set the following (Note: for development use `http://localhost:3000` instead of `https://yourspoke.example.com`):
+4. In your Auth0 app settings, set the following (Note: for development use `http://localhost:3000` instead of `https://yourspoke.example.com`. If following [Instructions for One-Click Deployment to Heroku](https://github.com/MoveOnOrg/Spoke/blob/main/docs/HOWTO_HEROKU_DEPLOY.md), use `https://<YOUR SPOKE APP>.herokuapp.com`.):
     + **Allowed Callback URLs** - `https://yourspoke.example.com/login-callback`
     + **Allowed Web Origins** - `https://yourspoke.example.com`
     + **Allowed Logout URLs** - `https://yourspoke.example.com/logout-callback`

docs/HOWTO_INTEGRATE_TWILIO.md

@@ -28,9 +28,17 @@ If you are using these instructions for an Heroku instance or AWS Lambda instanc
   - In your .env file, set `TWILIO_STATUS_CALLBACK_URL` to this same URL
 9. Click `Save`, and then visit the [dashboard](https://www.twilio.com/console)
 10. Under `Account Summary`
-  - `TWILIO_API_KEY` in your .env file (or Heroku config variable) is `ACCOUNT SID` in your Twilio console
-  - `TWILIO_APPLICATION_SID` in your .env file (or Heroku config variable) is `TWILIO_MESSAGE_SERVICE_SID` (these are the same values)
+  - `TWILIO_ACCOUNT_SID` in your .env file (or Heroku config variable) is `ACCOUNT SID` in your Twilio console
   - `TWILIO_AUTH_TOKEN` in your .env file (or Heroku config variable) is `AUTH TOKEN` in your Twilio console
 11. In your .env file, set `DEFAULT_SERVICE` to `twilio`
 12. If you want to send live text messages as part of your testing, you must buy a phone number and attach it to your project.
   - Click on `Numbers` and press on `+`. Search for a area code and click on buy (trial Twilio accounts give you $15~ to work with). In order to send messages, you will have to connect a credit card with a minimum charge of $20. 
+
+## Multi-Org Twilio Setup
+If you follow the instructions above, every organization and campaign in your instance will use the same Twilio account and the same messaging service (phone number pool). If you want to use different twilio accounts and/or messaging services for each organization, you can basically follow the same instuctions with a couple of tweaks.
+
+- In your .env file, set `TWILIO_MULTI_ORG` to `true`. This will enable an additional section on the organization settings page where you can set Twilio credentials for the organization.
+- For security, Twilio Auth Tokens are encrypted using the `SESSION_SECRET` environment variable before being stored in the database.
+- You can still set instance-wide credentials in the .env file (as described above). If you do, those credentials will be used as fallback if credentials aren't configured for an organization.
+- It is not required to configure all settings for all organizations. For example, to use a single site-wide Twilio account but with separate phone number pools for some organizations, follow the instuctions above and then set the Default Message Service SID (leaving the other fields blank) in the organizations settings for the orgs you want to override.
+- When using multiple Twilio accounts you will need to change the Inbound Request URL for your messaging service in the Twilio console [step 7 above]. It should look like `https://<YOUR_HEROKU_APP_URL>/twilio/<ORG_ID>`. The correct URL to use will be displayed on the settings page after you save the Twilio credentials.

docs/REFERENCE-environment_variables.md

@@ -77,10 +77,11 @@
 | SUPPRESS_SELF_INVITE                             | Boolean value to prevent self-invitations. Recommend setting before making sites available to public. _Default_: false.                                                                                                                                                                                                                                                                     |
 | SUPPRESS_DATABASE_AUTOCREATE                     | Suppress database auto-creation on first start. Mostly just used for test context                                                                                                                                                                                                                                                                                                           |
 | TERMS_REQUIRE                                    | Require texters to accept the [Terms page](../src/containers/Terms.jsx#L85) before they can start texting. _Default_: false                                                                                                                                                                                                                                                                 |
-| TWILIO_API_KEY                                   | Twilio API key. Required if using Twilio.                                                                                                                                                                                                                                                                                                                                                   |
-| TWILIO_APPLICATION_SID                           | Twilio application ID. Required if using Twilio.                                                                                                                                                                                                                                                                                                                                            |
-| TWILIO_AUTH_TOKEN                                | Twilio auth token. Required if using Twilio.                                                                                                                                                                                                                                                                                                                                                |
-| TWILIO_MESSAGE_SERVICE_SID                       | Twilio message service ID. Required if using Twilio.                                                                                                                                                                                                                                                                                                                                        |
+| TWILIO_ACCOUNT_SID                               | Global Twilio account SID. Required if using Twilio and `TWILIO_MULTI_ORG` is not set.                                                                                                                                                                                                                                                                                                      |
+| TWILIO_API_KEY                                   | _(Deprecated)_ Replaced by `TWILIO_ACCOUNT_SID`                                                                                                                                                                                                                                                                                                                                             |
+| TWILIO_AUTH_TOKEN                                | Global Twilio auth token. Required if using Twilio and `TWILIO_MULTI_ORG` is not set.                                                                                                                                                                                                                                                                                                       |
+| TWILIO_MESSAGE_SERVICE_SID                       | Global Twilio message service ID. Required if using Twilio and `TWILIO_MULTI_ORG` is not set.                                                                                                                                                                                                                                                                                               |
+| TWILIO_MULTI_ORG                                 | Boolean value to indicate if organizations can override Twilio credentials in the organization settings. _Default_: false.                                                                                                                                                                                                                                                                  |
 | TWILIO_STATUS_CALLBACK_URL                       | URL for Twilio status callbacks. Should end with `/twilio-message-report`, e.g. `https://example.org/twilio-message-report`. Required if using Twilio.                                                                                                                                                                                                                                      |
 | TWILIO_SQS_QUEUE_URL                             | AWS SQS URL to handle incoming messages when app isn't connected to twilio                                                                                                                                                                                                                                                                                                                  |
 | WAREHOUSE*DB*{TYPE,HOST,PORT,NAME,USER,PASSWORD} | Enables ability to load contacts directly from a SQL query from a separate data-warehouse db -- only is_superadmin-marked users will see the interface                                                                                                                                                                                                                                      |

jest.config.js

@@ -22,7 +22,7 @@ module.exports = {
     DATABASE_SETUP_TEARDOWN_TIMEOUT: 60000,
     PASSPORT_STRATEGY: "local",
     SESSION_SECRET: "it is JUST a test! -- it better be!",
-    TWILIO_API_KEY: "", // purposefully blank
+    TWILIO_ACCOUNT_SID: "", // purposefully blank
     TEST_ENVIRONMENT: "1",
     TEST_ENVIRONMENT_FAKE: "0",
     TEST_ENVIRONMENT_FAKE2: "false"

src/api/organization.js

@@ -18,5 +18,9 @@ export const schema = gql`
     textingHoursStart: Int
     textingHoursEnd: Int
     cacheable: Int
+    twilioAccountSid: String
+    twilioAuthToken: String
+    twilioMessageServiceSid: String
+    fullyConfigured: Boolean
   }
 `;

src/api/schema.js

@@ -290,6 +290,12 @@ const rootSchema = gql`
       organizationId: String!
       optOutMessage: String!
     ): Organization
+    updateTwilioAuth(
+      organizationId: String!
+      twilioAccountSid: String
+      twilioAuthToken: String
+      twilioMessageServiceSid: String
+    ): Organization
     bulkSendMessages(assignmentId: Int!): [CampaignContact]
     sendMessage(
       message: MessageInput!