Skip to content

Commit

Permalink
Merge branch 'aws-amplify:dev' into dev
Browse files Browse the repository at this point in the history
  • Loading branch information
@kuhlmanp
kuhlmanp authored Jan 18, 2023
2 parents 767ece2 + 09bfa2b commit 9ebfafb
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 92 deletions.
55 changes: 28 additions & 27 deletions packages/amplify-cli/src/__tests__/utils/encrypt-buffer.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,35 @@
import { encryptBuffer, encryptKey } from '../../commands/helpers/encryption-helpers'
import { encryptBuffer, encryptKey } from '../../commands/helpers/encryption-helpers';
import { v4 } from 'uuid';
import crypto from 'crypto'
describe("encryption helper", () => {
it("tests encryption helper", async() =>{
const originalKey = v4();
const plainText = crypto.randomBytes(10);
const encryptedText = await encryptBuffer(plainText, originalKey);
const bData = Buffer.from(encryptedText, 'base64');
import crypto from 'crypto';
describe('encryption helper', () => {
it('tests encryption helper', async () => {
const originalKey = v4();
const plainText = crypto.randomBytes(10);
const encryptedText = await encryptBuffer(plainText, originalKey);
const bData = Buffer.from(encryptedText, 'base64');

// convert data to buffers
const salt = bData.slice(0, 64);
const iv = bData.slice(64, 80);
const tag = bData.slice(80, 96);
const text = bData.slice(96);
// convert data to buffers
const salt = bData.slice(0, 64);
const iv = bData.slice(64, 80);
const tag = bData.slice(80, 96);
const text = bData.slice(96);

const masterkey = Buffer.from(originalKey, 'utf-8');
const masterkey = Buffer.from(originalKey, 'utf-8');

// derive key using; 32 byte key length
const key = crypto.pbkdf2Sync(masterkey, salt , 2145, 32, 'sha512');
// derive key using; 32 byte key length
const key = crypto.pbkdf2Sync(masterkey, salt, 2145, 32, 'sha512');

// AES 256 GCM Mode
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
decipher.setAuthTag(tag);
// AES 256 GCM Mode
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
decipher.setAuthTag(tag);

// encrypt the given text
const decrypted = decipher.update(text, 'binary', 'utf8') + decipher.final('utf8');
expect(decrypted).toEqual(plainText.toString('utf8'));
});
// encrypt the given text
const decrypted = decipher.update(text, 'binary', 'utf8') + decipher.final('utf8');
expect(decrypted).toEqual(plainText.toString('utf8'));
});

it("test encryption key", async () => {
expect(await encryptKey(v4())).toBeTruthy();
})
})
it('test encryption key', async () => {
jest.setTimeout(10000);
expect(await encryptKey(v4())).toBeTruthy();
});
});
4 changes: 2 additions & 2 deletions packages/amplify-dynamodb-simulator/__test__/index.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const ddbSimulator = require('..');
const fs = require('fs-extra');

jest.setTimeout(90 * 1000);
jest.mock('amplify-cli-core', () => ({
pathManager: {
getAmplifyPackageLibDirPath: jest.fn().mockReturnValue('./'),
Expand Down Expand Up @@ -41,8 +42,7 @@ describe('emulator operations', () => {
if (fs.existsSync(dbPath)) {
try {
fs.removeSync(dbPath);
}
catch(err) {
} catch (err) {
console.log(err);
}
}
Expand Down
89 changes: 59 additions & 30 deletions packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ const DELETE_LIMITS = {
PER_BATCH: {
OTHER: 50,
CFN_STACK: 100,
}
}
},
};

const reportPath = path.normalize(path.join(__dirname, '..', 'amplify-e2e-reports', 'stale-resources.json'));

Expand Down Expand Up @@ -140,19 +140,28 @@ const handleExpiredTokenException = (): void => {
process.exit();
};

/**
* Check if a resource is stale based on its created date
* @param created
* @returns
*/
const isStale = (created: Date): boolean => {
const now = new Date().getTime();
const isStale = now - created.getTime() > STALE_DURATION_MS;
return isStale;
};

/**
* We define a resource as viable for deletion if it matches TEST_REGEX in the name, and if it is > STALE_DURATION_MS old.
*/
const testBucketStalenessFilter = (resource: aws.S3.Bucket): boolean => {
const isTestResource = resource.Name.match(BUCKET_TEST_REGEX);
const isStaleResource = new Date().getUTCMilliseconds() - resource.CreationDate.getUTCMilliseconds() > STALE_DURATION_MS;
return isTestResource && isStaleResource;
return isTestResource && isStale(resource.CreationDate);
};

const testRoleStalenessFilter = (resource: aws.IAM.Role): boolean => {
const isTestResource = resource.RoleName.match(IAM_TEST_REGEX);
const isStaleResource = new Date().getUTCMilliseconds() - resource.CreateDate.getUTCMilliseconds() > STALE_DURATION_MS;
return isTestResource && isStaleResource;
return isTestResource && isStale(resource.CreateDate);
};

const testAppSyncApiStalenessFilter = (resource: aws.AppSync.GraphqlApi): boolean => {
Expand All @@ -161,15 +170,14 @@ const testAppSyncApiStalenessFilter = (resource: aws.AppSync.GraphqlApi): boolea
let isStaleResource = true;
if (createTimeTagValue) {
const createTime = new Date(createTimeTagValue);
isStaleResource = (new Date().getUTCMilliseconds() - createTime.getUTCMilliseconds()) > STALE_DURATION_MS;
isStaleResource = isStale(createTime);
}
return isTestResource && isStaleResource;
};

const testPinpointAppStalenessFilter = (resource: aws.Pinpoint.ApplicationResponse): boolean => {
const isTestResource = resource.Name.match(PINPOINT_TEST_REGEX);
const isStaleResource = new Date().getUTCMilliseconds() - new Date(resource.CreationDate).getUTCMilliseconds() > STALE_DURATION_MS;
return isTestResource && isStaleResource;
return isTestResource && isStale(new Date(resource.CreationDate));
};

/**
Expand Down Expand Up @@ -250,14 +258,17 @@ const getAWSConfig = ({ accessKeyId, secretAccessKey, sessionToken }: AWSAccount
* @returns Promise<AmplifyAppInfo[]> a list of Amplify Apps in the region with build info
*/
const getAmplifyApps = async (account: AWSAccountInfo, region: string): Promise<AmplifyAppInfo[]> => {
if(region === 'us-east-1' && account.parent){
if (region === 'us-east-1' && account.parent) {
return []; // temporarily disabled until us-east-1 is re-enabled for this account
}
const amplifyClient = new aws.Amplify(getAWSConfig(account, region));
try {
const amplifyApps = await amplifyClient.listApps({ maxResults: 25 }).promise(); // keeping it to 25 as max supported is 25
const result: AmplifyAppInfo[] = [];
for (const app of amplifyApps.apps) {
if (!isStale(app.createTime)) {
continue; // skip
}
const backends: Record<string, StackInfo> = {};
try {
const backendEnvironments = await amplifyClient.listBackendEnvironments({ appId: app.appId, maxResults: 5 }).promise();
Expand All @@ -278,7 +289,7 @@ const getAmplifyApps = async (account: AWSAccountInfo, region: string): Promise<
});
}
return result;
} catch (e){
} catch (e) {
console.log(e);
return [];
}
Expand Down Expand Up @@ -347,15 +358,15 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
StackStatusFilter: stackStatusFilter,
})
.promise();
// loop
// loop
let nextToken = stacks.NextToken;
while (nextToken && stacks.StackSummaries.length < DELETE_LIMITS.PER_REGION.CFN_STACK) {
const nextPage = await cfnClient
.listStacks({
StackStatusFilter: stackStatusFilter,
NextToken: nextToken,
})
.promise();
.promise();
stacks.StackSummaries.push(...nextPage.StackSummaries);
nextToken = nextPage.NextToken;
}
Expand All @@ -364,8 +375,14 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
// NOTE: every few months, we should disable the filter , and clean up all stacks (not just root stacks)
// this is because some child stacks fail to delete (but we don't let that stop us from deleting root stacks)
// eventually, we must clean up those child stacks too.
let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
if(rootStacks.length > DELETE_LIMITS.PER_REGION.CFN_STACK){
let rootStacks = stacks.StackSummaries.filter(stack => {
const isRoot = !stack.RootId;
if (!isStale(stack.CreationTime)) {
console.log('Skipping stack because created date is:', stack.CreationTime);
}
return isRoot && isStale;
});
if (rootStacks.length > DELETE_LIMITS.PER_REGION.CFN_STACK) {
// we can only delete 100 stacks accross all regions every batch,
// so we shouldn't take more than 50 stacks from each of those 8 regions.
// this should at least limit calls to getStackDetails below
Expand Down Expand Up @@ -685,9 +702,7 @@ const deleteAppSyncApis = async (account: AWSAccountInfo, accountIndex: number,
};

const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, api: AppSyncApiInfo): Promise<void> => {
const {
apiId, name, region,
} = api;
const { apiId, name, region } = api;
try {
console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name}`);
const appSync = new aws.AppSync(getAWSConfig(account, region));
Expand Down Expand Up @@ -755,12 +770,16 @@ const deleteResources = async (
}

if (resources.pinpointApps) {
console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.pinpointApps.length} pinpoint apps on ACCOUNT[${accountIndex}]`);
console.log(
`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.pinpointApps.length} pinpoint apps on ACCOUNT[${accountIndex}]`,
);
await deletePinpointApps(account, accountIndex, Object.values(resources.pinpointApps));
}

if (resources.appSyncApis) {
console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.appSyncApis.length} appSyncApis on ACCOUNT[${accountIndex}]`);
console.log(
`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.appSyncApis.length} appSyncApis on ACCOUNT[${accountIndex}]`,
);
await deleteAppSyncApis(account, accountIndex, Object.values(resources.appSyncApis));
}
}
Expand Down Expand Up @@ -876,13 +895,21 @@ const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, fil
const orphanAppSyncApis = (await Promise.all(orphanAppSyncApisPromise)).flat();

const allResources = mergeResourcesByCCIJob(
apps, stacks, buckets, orphanBuckets, orphanIamRoles, orphanPinpointApplications, orphanAppSyncApis
apps,
stacks,
buckets,
orphanBuckets,
orphanIamRoles,
orphanPinpointApplications,
orphanAppSyncApis,
);
// cleanup resources that are <unknown> but that are definitely amplify resources
// this includes apps with names that include "test" or stacks that include both "amplify" & "test"
const testApps = allResources["<unknown>"].amplifyApps?.filter(a => a.name.toLocaleLowerCase().includes('test'));
const testStacks = allResources["<unknown>"].stacks?.filter(s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'));
const orphanedResources = allResources["<orphan>"];
const testApps = allResources['<unknown>'].amplifyApps?.filter(a => a.name.toLocaleLowerCase().includes('test'));
const testStacks = allResources['<unknown>'].stacks?.filter(
s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'),
);
const orphanedResources = allResources['<orphan>'];
orphanedResources.amplifyApps = orphanedResources.amplifyApps ?? [];
orphanedResources.stacks = orphanedResources.stacks ?? [];
orphanedResources.amplifyApps.push(...(testApps ? testApps : []));
Expand Down Expand Up @@ -922,12 +949,14 @@ const cleanup = async (): Promise<void> => {

const filterPredicate = getFilterPredicate(args);
const accounts = await getAccountsToCleanup();
for(let i = 0 ;i < 3; i ++){
console.log("CLEANUP ROUND: ", i + 1);
await Promise.all(accounts.map((account, i) => {
return cleanupAccount(account, i, filterPredicate);
}));
await sleep(60 * 1000);// run again after 60 seconds
for (let i = 0; i < 3; i++) {
console.log('CLEANUP ROUND: ', i + 1);
await Promise.all(
accounts.map((account, i) => {
return cleanupAccount(account, i, filterPredicate);
}),
);
await sleep(60 * 1000); // run again after 60 seconds
}
console.log('Done cleaning all accounts!');
};
Expand Down
Loading

0 comments on commit 9ebfafb

Please sign in to comment.