Merge pull request coollabsio#571 from coollabsio/next

v3.8.4

.gitignore

@@ -11,4 +11,5 @@ dist
 client
 apps/api/db/*.db
 local-serve
-apps/api/db/migration.db-journal
+apps/api/db/migration.db-journal
+apps/api/core*

apps/api/src/index.ts

@@ -161,7 +161,7 @@ async function initServer() {
 	} catch (error) { }
 	try {
 		const isOlder = compareVersions('3.8.1', version);
-		if (isOlder === -1) {
+		if (isOlder === 1) {
 			await prisma.build.updateMany({ where: { status: { in: ['running', 'queued'] } }, data: { status: 'failed' } });
 		}
 	} catch (error) { }

apps/api/src/jobs/deployApplication.ts

@@ -4,7 +4,7 @@ import fs from 'fs/promises';
 import yaml from 'js-yaml';
 
 import { copyBaseConfigurationFiles, makeLabelForStandaloneApplication, saveBuildLog, setDefaultConfiguration } from '../lib/buildPacks/common';
-import { createDirectories, decrypt, defaultComposeConfiguration, executeDockerCmd, getDomain, prisma } from '../lib/common';
+import { createDirectories, decrypt, defaultComposeConfiguration, executeDockerCmd, getDomain, prisma, decryptApplication } from '../lib/common';
 import * as importers from '../lib/importers';
 import * as buildpacks from '../lib/buildPacks';
 
@@ -27,7 +27,7 @@ import * as buildpacks from '../lib/buildPacks';
 
 		const th = throttle(async () => {
 			try {
-				const queuedBuilds = await prisma.build.findMany({ where: { status: 'queued' }, orderBy: { createdAt: 'asc' } });
+				const queuedBuilds = await prisma.build.findMany({ where: { status: { in: ['queued', 'running'] } }, orderBy: { createdAt: 'asc' } });
 				const { concurrentBuilds } = await prisma.setting.findFirst({})
 				if (queuedBuilds.length > 0) {
 					parentPort.postMessage({ deploying: true });
@@ -37,68 +37,72 @@ import * as buildpacks from '../lib/buildPacks';
 
 					for (const queueBuild of queuedBuilds) {
 						actions.push(async () => {
-							const application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } })
-							const { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, forceRebuild } = queueBuild
-							const {
-								id: applicationId,
-								repository,
-								name,
-								destinationDocker,
-								destinationDockerId,
-								gitSource,
-								configHash,
-								fqdn,
-								projectId,
-								secrets,
-								phpModules,
-								settings,
-								persistentStorage,
-								pythonWSGI,
-								pythonModule,
-								pythonVariable,
-								denoOptions,
-								exposePort,
-								baseImage,
-								baseBuildImage,
-								deploymentType,
-							} = application
-							let {
-								branch,
-								buildPack,
-								port,
-								installCommand,
-								buildCommand,
-								startCommand,
-								baseDirectory,
-								publishDirectory,
-								dockerFileLocation,
-								denoMainFile
-							} = application
-							const currentHash = crypto
-								.createHash('sha256')
-								.update(
-									JSON.stringify({
-										pythonWSGI,
-										pythonModule,
-										pythonVariable,
-										deploymentType,
-										denoOptions,
-										baseImage,
-										baseBuildImage,
-										buildPack,
-										port,
-										exposePort,
-										installCommand,
-										buildCommand,
-										startCommand,
-										secrets,
-										branch,
-										repository,
-										fqdn
-									})
-								)
-								.digest('hex');
+							let application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } })
+							let { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, forceRebuild } = queueBuild
+							application = decryptApplication(application)
 							try {
+								if (queueBuild.status === 'running') {
+									await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
+								}
+								const {
+									id: applicationId,
+									repository,
+									name,
+									destinationDocker,
+									destinationDockerId,
+									gitSource,
+									configHash,
+									fqdn,
+									projectId,
+									secrets,
+									phpModules,
+									settings,
+									persistentStorage,
+									pythonWSGI,
+									pythonModule,
+									pythonVariable,
+									denoOptions,
+									exposePort,
+									baseImage,
+									baseBuildImage,
+									deploymentType,
+								} = application
+								let {
+									branch,
+									buildPack,
+									port,
+									installCommand,
+									buildCommand,
+									startCommand,
+									baseDirectory,
+									publishDirectory,
+									dockerFileLocation,
+									denoMainFile
+								} = application
+								const currentHash = crypto
+									.createHash('sha256')
+									.update(
+										JSON.stringify({
+											pythonWSGI,
+											pythonModule,
+											pythonVariable,
+											deploymentType,
+											denoOptions,
+											baseImage,
+											baseBuildImage,
+											buildPack,
+											port,
+											exposePort,
+											installCommand,
+											buildCommand,
+											startCommand,
+											secrets,
+											branch,
+											repository,
+											fqdn
+										})
+									)
+									.digest('hex');
 								const { debug } = settings;
 								if (concurrency === 1) {
 									await prisma.build.updateMany({
@@ -258,7 +262,6 @@ import * as buildpacks from '../lib/buildPacks';
 									];
 									if (secrets.length > 0) {
 										secrets.forEach((secret) => {
-											secret.value = decrypt(secret.value)
 											if (pullmergeRequestId) {
 												if (secret.isPRMRSecret) {
 													envs.push(`${secret.name}=${secret.value}`);
@@ -353,13 +356,16 @@ import * as buildpacks from '../lib/buildPacks';
 									where: { id: buildId, status: { in: ['queued', 'running'] } },
 									data: { status: 'failed' }
 								});
-								await saveBuildLog({ line: error, buildId, applicationId });
+								await saveBuildLog({ line: error, buildId, applicationId: application.id });
 							}
 						});
+
 					}
+
 					await pAll.default(actions, { concurrency })
 				}
 			} catch (error) {
+				console.log(error)
 			} finally {
 			}
 		})

apps/api/src/lib/common.ts

@@ -19,7 +19,7 @@ import * as serviceFields from './serviceFields'
 import { saveBuildLog } from './buildPacks/common';
 import { scheduler } from './scheduler';
 
-export const version = '3.8.3';
+export const version = '3.8.4';
 export const isDev = process.env.NODE_ENV === 'development';
 
 const algorithm = 'aes-256-ctr';
@@ -1977,6 +1977,12 @@ export async function cleanupDockerStorage(dockerId, lowDiskSpace, force) {
 		} catch (error) {
 			//console.log(error);
 		}
+		// Cleanup build caches
+		try {
+			await executeDockerCmd({ dockerId, command: `docker builder prune -a -f` })
+		} catch (error) {
+			//console.log(error);
+		}
 	}
 }
 
@@ -2022,3 +2028,27 @@ export function defaultComposeConfiguration(network: string): any {
 		}
 	}
 }
+export function decryptApplication(application: any) {
+    if (application) {
+        if (application?.gitSource?.githubApp?.clientSecret) {
+            application.gitSource.githubApp.clientSecret = decrypt(application.gitSource.githubApp.clientSecret) || null;
+        }
+        if (application?.gitSource?.githubApp?.webhookSecret) {
+            application.gitSource.githubApp.webhookSecret = decrypt(application.gitSource.githubApp.webhookSecret) || null;
+        }
+        if (application?.gitSource?.githubApp?.privateKey) {
+            application.gitSource.githubApp.privateKey = decrypt(application.gitSource.githubApp.privateKey) || null;
+        }
+        if (application?.gitSource?.gitlabApp?.appSecret) {
+            application.gitSource.gitlabApp.appSecret = decrypt(application.gitSource.gitlabApp.appSecret) || null;
+        }
+        if (application?.secrets.length > 0) {
+            application.secrets = application.secrets.map((s: any) => {
+                s.value = decrypt(s.value) || null
+                return s;
+            });
+        }
+
+        return application;
+    }
+}

apps/api/src/routes/api/v1/applications/handlers.ts

@@ -451,6 +451,7 @@ export async function deployApplication(request: FastifyRequest<DeployApplicatio
                 data: {
                     id: buildId,
                     applicationId: id,
+                    sourceBranch: branch,
                     branch: application.branch,
                     pullmergeRequestId,
                     forceRebuild,

apps/api/src/routes/api/v1/iam/handlers.ts

@@ -158,8 +158,11 @@ export async function getTeam(request: FastifyRequest<OnlyId>, reply: FastifyRep
         });
         const team = await prisma.team.findUnique({ where: { id }, include: { permissions: true } });
         const invitations = await prisma.teamInvitation.findMany({ where: { teamId: team.id } });
+        const { teams } = await prisma.user.findUnique({ where: { id: userId }, include: { teams: true } })
         return {
+            currentTeam: teamId,
             team,
+            teams,
             permissions,
             invitations
         };
@@ -275,10 +278,10 @@ export async function inviteToTeam(request: FastifyRequest<InviteToTeam>, reply:
         if (!userFound) {
             throw {
                 message: `No user found with '${email}' email address.`
-            };         
+            };
         }
         const uid = userFound.id;
-        if (uid === userId) {          
+        if (uid === userId) {
             throw {
                 message: `Invitation to yourself? Whaaaaat?`
             };

apps/api/src/routes/api/v1/sources/handlers.ts

@@ -9,7 +9,7 @@ export async function listSources(request: FastifyRequest) {
     try {
         const teamId = request.user?.teamId;
         const sources = await prisma.gitSource.findMany({
-            where: { teams: { some: { id: teamId === '0' ? undefined : teamId } } },
+            where: { teams: { some: { id: teamId === '0' ? undefined : teamId } }, githubApp: { gitSource: { forPublic: false } } },
             include: { teams: true, githubApp: true, gitlabApp: true }
         });
         return {

apps/api/src/routes/webhooks/github/handlers.ts

@@ -146,7 +146,7 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
                     message: 'Queued. Thank you!'
                 };
             } else if (githubEvent === 'pull_request') {
-                const pullmergeRequestId = body.number;
+                const pullmergeRequestId = body.number.toString();
                 const pullmergeRequestAction = body.action;
                 const sourceBranch = body.pull_request.head.ref.includes('/') ? body.pull_request.head.ref.split('/')[2] : body.pull_request.head.ref;
                 if (!allowedActions.includes(pullmergeRequestAction)) {

apps/ui/src/lib/components/Usage.svelte

@@ -86,8 +86,8 @@
 
 <div class="w-full">
 	<div class="flex items-center">
-		<h1 class="title text-4xl">Hardware Details</h1>
-		<div class="flex  space-x-4">
+		<h1 class="title lg:text-3xl">Hardware Details</h1>
+		<div class="flex space-x-4">
 			{#if $appSession.teamId === '0'}
 				<button on:click={manuallyCleanupStorage} class:loading={loading.cleanup} class="btn btn-sm"
 					>Cleanup Storage</button
@@ -101,42 +101,41 @@
 		</div>
 	</div>
 	<div class="divider" />
-	<div class="grid grid-flow-col gap-4 grid-rows-3 lg:grid-rows-1">
-		<div class="stats stats-vertical lg:stats-horizontal w-full mb-5 bg-transparent rounded">
-			<div class="font-bold flex lg:justify-center">Memory</div>
+	<div class="grid grid-flow-col gap-4 grid-rows-3 justify-start lg:justify-center lg:grid-rows-1">
+		<div class="stats stats-vertical min-w-[16rem] mb-5 rounded bg-transparent">
 			<div class="stat">
-				<div class="stat-title">Total</div>
+				<div class="stat-title">Total Memory</div>
 				<div class="stat-value text-2xl">
 					{(usage?.memory.totalMemMb).toFixed(0)}<span class="text-sm">MB</span>
 				</div>
 			</div>
 
 			<div class="stat">
-				<div class="stat-title">Used</div>
+				<div class="stat-title">Used Memory</div>
 				<div class="stat-value text-2xl">
 					{(usage?.memory.usedMemMb).toFixed(0)}<span class="text-sm">MB</span>
 				</div>
 			</div>
 
 			<div class="stat">
-				<div class="stat-title">Free</div>
+				<div class="stat-title">Free Memory</div>
 				<div class="stat-value text-2xl">
 					{usage?.memory.freeMemPercentage}<span class="text-sm">%</span>
 				</div>
 			</div>
 		</div>
 
-		<div class="stats stats-vertical lg:stats-horizontal w-full mb-5 bg-transparent rounded">
-			<div class="font-bold flex lg:justify-center">CPU</div>
+		<div class="stats stats-vertical min-w-[20rem] mb-5 bg-transparent rounded">
+
 			<div class="stat">
-				<div class="stat-title">Total</div>
+				<div class="stat-title">Total CPU</div>
 				<div class="stat-value text-2xl">
 					{usage?.cpu.count}
 				</div>
 			</div>
 
 			<div class="stat">
-				<div class="stat-title">Usage</div>
+				<div class="stat-title">CPU Usage</div>
 				<div class="stat-value text-2xl">
 					{usage?.cpu.usage}<span class="text-sm">%</span>
 				</div>
@@ -147,24 +146,23 @@
 				<div class="stat-value text-2xl">{usage?.cpu.load}</div>
 			</div>
 		</div>
-		<div class="stats stats-vertical lg:stats-horizontal w-full mb-5 bg-transparent rounded">
-			<div class="font-bold flex lg:justify-center">Disk</div>
+		<div class="stats stats-vertical min-w-[16rem] mb-5 bg-transparent rounded">
 			<div class="stat">
-				<div class="stat-title">Total</div>
+				<div class="stat-title">Total Disk</div>
 				<div class="stat-value text-2xl">
 					{usage?.disk.totalGb}<span class="text-sm">GB</span>
 				</div>
 			</div>
 
 			<div class="stat">
-				<div class="stat-title">Used</div>
+				<div class="stat-title">Used Disk</div>
 				<div class="stat-value text-2xl">
 					{usage?.disk.usedGb}<span class="text-sm">GB</span>
 				</div>
 			</div>
 
 			<div class="stat">
-				<div class="stat-title">Free</div>
+				<div class="stat-title">Free Disk</div>
 				<div class="stat-value text-2xl">
 					{usage?.disk.freePercentage}<span class="text-sm">%</span>
 				</div>

apps/ui/src/lib/components/svg/applications/Docker.svelte

@@ -3,7 +3,7 @@
 </script>
 
 
-<svg viewBox="0 0 128 128" class={isAbsolute ? 'absolute top-0 left-0 -m-8 h-16 w-16' : 'mx-auto w-10 h-10'}>
+<svg viewBox="0 0 128 128" class={isAbsolute ? 'absolute top-0 left-0 -m-5 h-12 w-12' : 'mx-auto w-10 h-10'}>
 	<path d="M124.8 52.1c-4.3-2.5-10-2.8-14.8-1.4-.6-5.2-4-9.7-8-12.9l-1.6-1.3-1.4 1.6c-2.7 3.1-3.5 8.3-3.1 12.3.3 2.9 1.2 5.9 3 8.3-1.4.8-2.9 1.9-4.3 2.4-2.8 1-5.9 2-8.9 2H79V49H66V24H51v12H26v13H13v14H1.8l-.2 1.5c-.5 6.4.3 12.6 3 18.5l1.1 2.2.1.2c7.9 13.4 21.7 19 36.8 19 29.2 0 53.3-13.1 64.3-40.6 7.4.4 15-1.8 18.6-8.9l.9-1.8-1.6-1zM28 39h10v11H28V39zm13.1 44.2c0 1.7-1.4 3.1-3.1 3.1-1.7 0-3.1-1.4-3.1-3.1 0-1.7 1.4-3.1 3.1-3.1 1.7.1 3.1 1.4 3.1 3.1zM28 52h10v11H28V52zm-13 0h11v11H15V52zm27.7 50.2c-15.8-.1-24.3-5.4-31.3-12.4 2.1.1 4.1.2 5.9.2 1.6 0 3.2 0 4.7-.1 3.9-.2 7.3-.7 10.1-1.5 2.3 5.3 6.5 10.2 14 13.8h-3.4zM51 63H40V52h11v11zm0-13H40V39h11v11zm13 13H53V52h11v11zm0-13H53V39h11v11zm0-13H53V26h11v11zm13 26H66V52h11v11zM38.8 81.2c-.2-.1-.5-.2-.8-.2-1.2 0-2.2 1-2.2 2.2 0 1.2 1 2.2 2.2 2.2s2.2-1 2.2-2.2c0-.3-.1-.6-.2-.8-.2.3-.4.5-.8.5-.5 0-.9-.4-.9-.9.1-.4.3-.7.5-.8z" fill="#019BC6"></path>
 	</svg>