First platform initialization can fail and not be detected/retried

[gileri]

Description

Environment

1. OS (where OpenCTI server runs): official docker image
2. OpenCTI version: 6.4.9
3. OpenCTI client: frontend
4. Other environment details:

Reproducible Steps

Steps to create the smallest reproducible scenario:

1. Start a new OCTI platform
2. Have some issue stopping initialization
3. Platform crashes
4. Start the platform again

Expected Output

• Best case, it retries the initialization
• Alternatively, it fails with an error explaining what failed in the initialization, to allow manual remediation

Actual Output

Initialization is considered successful, but Administrator role is not created for example.

2025-01-29 10:50:36.291	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T09:50:36.290Z","version":"6.4.9"}
2025-01-29 10:50:52.913	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T09:50:52.912Z","version":"6.4.9"}
2025-01-29 10:51:19.771	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T09:51:19.770Z","version":"6.4.9"}
2025-01-29 10:52:01.671	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T09:52:01.671Z","version":"6.4.9"}
2025-01-29 10:53:35.789	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T09:53:35.787Z","version":"6.4.9"}
2025-01-29 10:56:19.837	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T09:56:19.835Z","version":"6.4.9"}
2025-01-29 11:01:22.821	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T10:01:22.821Z","version":"6.4.9"}
2025-01-29 11:06:31.846	{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"Server not initialized yet, please try again.","name":"XMinioServerNotInitialized","stack":"XMinioServerNotInitialized: Server not initialized yet, please try again.\n    at throwDefaultError (/opt/opencti/build/back.js:543:7915)\n    at /opt/opencti/build/back.js:543:8239\n    at de_CommandError (/opt/opencti/build/back.js:857:255159)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at /opt/opencti/build/back.js:545:2131\n    at /opt/opencti/build/back.js:854:6190\n    at /opt/opencti/build/back.js:565:62216\n    at /opt/opencti/build/back.js:853:65252\n    at /opt/opencti/build/back.js:853:65812\n    at /opt/opencti/build/back.js:544:4600\n    at initializeBucket (/opt/opencti/build/back.js:2962:18985)\n    at storageInit (/opt/opencti/build/back.js:2962:19121)\n    at checkSystemDependencies (/opt/opencti/build/back.js:3553:27636)\n    at platformStart (/opt/opencti/build/back.js:3610:61740)"},"level":"error","message":"[OPENCTI] System dependencies check failed","source":"backend","timestamp":"2025-01-29T10:06:31.845Z","version":"6.4.9"}
2025-01-29 11:12:24.765	{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["platform_init_lock"]},"code":"LOCK_ERROR","message":"Platform cant get the lock for initialization (can be due to other instance currently migrating/initializing)","name":"LOCK_ERROR","stack":"GraphQLError: Platform cant get the lock for initialization (can be due to other instance currently migrating/initializing)\n    at error (/opt/opencti/build/back.js:2168:2273)\n    at LockTimeoutError (/opt/opencti/build/back.js:2168:5539)\n    at platformInit (/opt/opencti/build/back.js:3553:30505)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at platformStart (/opt/opencti/build/back.js:3610:62059)"},"level":"error","message":"[OPENCTI] Platform default initialization failed","source":"backend","timestamp":"2025-01-29T10:12:24.764Z","version":"6.4.9"}
2025-01-29 11:12:48.423	{"level":"info","message":"[MIGRATION] Read 0 migrations from the database","timestamp":"2025-01-29T10:12:48.422Z"}
2025-01-29 11:12:48.424	{"level":"info","message":"[MIGRATION] Platform already up to date, nothing to migrate","timestamp":"2025-01-29T10:12:48.424Z"}
2025-01-29 11:12:48.425	{"level":"info","message":"[MIGRATION] Migration process completed","timestamp":"2025-01-29T10:12:48.425Z"}

Additional information

One probable cause of the initialization failure was the loss of S3/Minio.

Looks like initializeMigration() is done early, before initializeAdminUser() for example, so if initializeMigration succeeds, but not initializeAdminUser(), platform won't create it again.

After initalizeMigration is done, the platform is considered correctly initialized.

Screenshots (optional)

[vel-atlassian]

In our case, we got this:
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["UUID","user--UUID"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:2168:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:2168:5539)\n at updateAttributeMetaResolved (/opt/opencti/build/back.js:2962:98440)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at initAdmin (/opt/opencti/build/back.js:2962:336672)\n at initializeAdminUser (/opt/opencti/build/back.js:2962:443119)\n at platformInit (/opt/opencti/build/back.js:3553:30245)\n at platformStart (/opt/opencti/build/back.js:3610:62059)"},"level":"error","message":"[OPENCTI] Platform default initialization failed","source":"backend","timestamp":"2025-01-30T21:49:35.530Z","version":"6.4.9"}