Save context in between tasks and fix relations migration (directus-l…

…abs#37)

* Add context functionality

This commit includes a new commandline option to pass a context file.
Moreover, it provides the basis to track which steps where already completed
through the context. There will always be a default context file that is used to seed the steps.

* Skipping steps based on completedSteps in context

Adding skip functions to skip a certain step if it is already completed in the passed context file

* Writing context after migrating schema steps

* Saving context after files migration

* Save context for roles and users

* Remove step for data as context isn't modified

* Adjust behavior of m2m fields during collection creation

Currently, m2m fields in v8 collections aren't correctly created, so that they work as expected in v9.

* Change the fieldtype to UUID if the field is pointing to directus_files_id

* Moving relations migration to own task

This commit includes separating the relations migration to its own task.
This is mainly done to avoid issues with migrating data due to foreign keys or other issuse with relations.
Secondly, this commit also includes an update to the format of relations, as v9 has moved relations data to a meta object.

Fix relations mapping to check in meta objects

Run relations migration before pushing data

The relations need to be added first, otherwise the items with relations of collections are not handled correctly by the API.
This commit also adds a final context saving.

fix gitignore

* Sort collections to add actual before junction collections

Sort collections based on their relations

This commit tries to implement some logic to sort collections based on their dependencies. Unfortunately, this is likely still error prone, especially for highly complex v8 instances with lots of relations between collections.
I think we should try to implement something more robust here.

* Run prettier on index.js

* Move context steps to a subfolder

This commits moves all generated context dumps to a dedicated folder.

* Add description for usage of context commandline option

* Separate getting of relations data from migration

This commit separatess the getting of v8 relations and saves them to a separate context file. This helps users debug potential failures during relation migration.

.gitignore

@@ -2,4 +2,8 @@ node_modules
 .DS_Store
 .vscode
 .env
-temp.json
+temp.json
+
+
+# Ignore context files except start
+context/state/*

context/start.json

@@ -0,0 +1,14 @@
+{
+  "completedSteps": {
+    "schema": false,
+    "collections": false,
+    "files": false,
+    "roles": false,
+    "users": false,
+    "relationsv8": false,
+    "relations": false,
+    "data": false,
+    "completed": false
+  },
+  "collectionsV9": []
+}

index.js

@@ -1,52 +1,106 @@
 import Listr from "listr";
 
 import commandLineArgs from "command-line-args";
+import * as fs from "fs";
 import { migrateSchema } from "./tasks/schema.js";
 import { migrateFiles } from "./tasks/files.js";
 import { migrateUsers } from "./tasks/users.js";
 import { migrateData } from "./tasks/data.js";
+import { migrateRelations } from "./tasks/relations.js";
 
 const commandLineOptions = commandLineArgs([
-	{
-		name: "skipCollections",
-		alias: "s",
-		type: String,
-		multiple: true,
-		defaultValue: [],
-	},
+  {
+    name: "skipCollections",
+    alias: "s",
+    type: String,
+    multiple: true,
+    defaultValue: [],
+  },
+  {
+    name: "useContext",
+    alias: "c",
+    type: String,
+    multiple: false,
+    defaultValue: "./context/start.json",
+  },
 ]);
 
 const tasks = new Listr([
-	{
-		title: "Migrating Schema",
-		task: (context) => {
-			context.skipCollections = commandLineOptions.skipCollections;
-			return migrateSchema(context);
-		},
-	},
-	{
-		title: "Migration Files",
-		task: migrateFiles,
-	},
-	{
-		title: "Migrating Users",
-		task: migrateUsers,
-	},
-	{
-		title: "Migrating Data",
-		task: migrateData,
-	},
+  {
+    title: "Loading context",
+    task: setupContext,
+  },
+  {
+    title: "Migrating Schema",
+    skip: (context) =>
+      context.completedSteps.schema === true &&
+      context.completedSteps.collections === true,
+    task: (context) => {
+      context.skipCollections = commandLineOptions.skipCollections;
+      return migrateSchema(context);
+    },
+  },
+  {
+    title: "Migration Files",
+    skip: (context) => context.completedSteps.files === true,
+    task: migrateFiles,
+  },
+  {
+    title: "Migrating Users",
+    skip: (context) =>
+      context.completedSteps.roles === true &&
+      context.completedSteps.users === true,
+    task: migrateUsers,
+  },
+  {
+    title: "Migrating Relations",
+    skip: (context) =>
+      context.completedSteps.relationsv8 === true &&
+      context.completedSteps.relations === true,
+    task: migrateRelations,
+  },
+  {
+    title: "Migrating Data",
+    skip: (context) => context.completedSteps.data === true,
+    task: migrateData,
+  },
+
+  {
+    title: "Save final context",
+    task: (context) => writeContext(context, "completed"),
+  },
 ]);
 
+export async function writeContext(context, section) {
+  context.completedSteps[section] = true;
+  await fs.promises.writeFile(
+    `./context/state/${section}.json`,
+    JSON.stringify(context)
+  );
+}
+
+async function setupContext(context) {
+  const contextJSON = await fs.promises.readFile(
+    commandLineOptions.useContext,
+    "utf8"
+  );
+  console.log("Loading context");
+  const fetchedContext = JSON.parse(contextJSON);
+  Object.entries(fetchedContext).forEach(([key, value]) => {
+    context[key] = value;
+  });
+  console.log(`✨ Loaded context succesfully`);
+}
+
 console.log(
-	`✨ Migrating ${process.env.V8_URL} (v8) to ${process.env.V9_URL} (v9)...`
+  `✨ Migrating ${process.env.V8_URL} (v8) to ${process.env.V9_URL} (v9)...`
 );
 
 tasks
-	.run()
-	.then(() => {
-		console.log("✨ All set! Migration successful.");
-	})
-	.catch((err) => {
-		console.error(err);
-	});
+  .run()
+  .then(() => {
+    console.log("✨ All set! Migration successful.");
+  })
+  .catch((err) => {
+    console.error(err);
+  });

readme.md

@@ -1,6 +1,6 @@
 # Migration Tool
 
-Migrate from a v8 instance to v9 instance. 
+Migrate from a v8 instance to v9 instance.
 
 This tool will copy over:
 
@@ -39,9 +39,17 @@ V9_TOKEN="admin"
 ```
 3) Run `npm install`
 4) Run the `index.js` file: `node index.js`
+
+
+### Commandline Options
 ***
-You can exclude collections/database tables from being migrated by using:
+You can exclude collections/database tables from being migrated by using the `-s` or `--skipCollections` flag:
 ```
 node index.js -s <table_name> <another_table_name>
 ```
 ***
+You can pass a context file to resume from a specific point or to modify the data manually that you want to use for the migration by using the `-c` or `--useContext` flag:
+```
+node index.js -c <path_to_context>
+```
+***

tasks/data.js

@@ -29,9 +29,38 @@ async function getCounts(context) {
 	}
 }
 
+// This is definitely a hack to achieve first adding items of collections that have dependencies in other collections i.e m2m, o2m
+// FIXME: Implement a more robust solution to sort collections based on their dependencies, or swap to a different way to seed the data
+function moveJunctionCollectionsBack(a,b) {
+	if (a.note === "Junction Collection" || b.note === "Junction Collection") {
+		if (a.note === "Junction Collection") {
+			return 1;
+		}
+
+		if (b.note === "Junction Collection") {
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+function moveManyToOne(a,b) {
+	if ( Object.values(a.fields).find(element => element.interface === 'many-to-one') ) {
+		return 1;
+	}
+
+	if ( Object.values(b.fields).find(element => element.interface === 'many-to-one') ) {
+		return -1;
+	}
+
+	return 0;
+}
+
 async function insertData(context) {
+	let sortedCollections = context.collections.sort(moveManyToOne).sort(moveJunctionCollectionsBack);
 	return new Listr(
-		context.collections.map((collection) => ({
+		sortedCollections.map((collection) => ({
 			title: collection.collection,
 			task: insertCollection(collection),
 		}))
@@ -72,8 +101,8 @@ async function insertBatch(collection, page, context, task) {
 
 	const systemRelationsForCollection = context.relations.filter((relation) => {
 		return (
-			relation.many_collection === collection.collection &&
-			relation.one_collection.startsWith("directus_")
+			relation?.meta?.many_collection === collection.collection &&
+			relation?.meta?.one_collection.startsWith("directus_")
 		);
 	});
 
@@ -82,12 +111,12 @@ async function insertBatch(collection, page, context, task) {
 			? recordsResponse.data.data
 			: recordsResponse.data.data.map((item) => {
 					for (const systemRelation of systemRelationsForCollection) {
-						if (systemRelation.one_collection === "directus_users") {
-							item[systemRelation.many_field] =
-								context.userMap[item[systemRelation.many_field]];
-						} else if (systemRelation.one_collection === "directus_files") {
-							item[systemRelation.many_field] =
-								context.fileMap[item[systemRelation.many_field]];
+						if (systemRelation?.meta?.one_collection === "directus_users") {
+							item[systemRelation?.meta?.many_field] =
+								context.userMap[item[systemRelation?.meta?.many_field]];
+						} else if (systemRelation?.meta?.one_collection === "directus_files") {
+							item[systemRelation?.meta?.many_field] =
+								context.fileMap[item[systemRelation?.meta?.many_field]];
 						}
 					}
 
@@ -102,6 +131,7 @@ async function insertBatch(collection, page, context, task) {
 		}
 	} catch (err) {
 		console.log(err.response.data);
+		throw Error("Data migration failed. Check directus logs for most insight.")
 	}
 }
 

tasks/files.js

@@ -1,5 +1,6 @@
 import Listr from "listr";
 import { apiV8, apiV9 } from "../api.js";
+import { writeContext } from "../index.js";
 
 export async function migrateFiles(context) {
 	return new Listr([
@@ -11,6 +12,10 @@ export async function migrateFiles(context) {
 			title: "Uploading Files",
 			task: uploadFiles,
 		},
+		{
+      title: "Saving context",
+      task: () => writeContext(context, "files")
+    },
 	]);
 }
 

tasks/relations.js

@@ -0,0 +1,84 @@
+import Listr from "listr";
+import { apiV8, apiV9 } from "../api.js";
+import { writeContext } from "../index.js";
+
+export async function migrateRelations(context) {
+  return new Listr([
+    {
+      title: "Get v8 Relations",
+      task: () => getRelationsData(context),
+      skip: (context) => context.completedSteps.relationsv8 === true,
+    },
+    {
+      title: "Saving Relations context",
+      task: () => writeContext(context, "relationsv8"),
+      skip: (context) => context.completedSteps.relationsv8 === true,
+    },
+    {
+      title: "Migrating Relations",
+      task: () => migrateRelationsData(context),
+      skip: (context) => context.completedSteps.relations === true,
+    },
+    {
+      title: "Saving Relations context",
+      task: () => writeContext(context, "relations"),
+      skip: (context) => context.completedSteps.relations === true,
+    },
+  ]);
+}
+
+async function getRelationsData(context) {
+  const relations = await apiV8.get("/relations", { params: { limit: -1 } });
+  context.relationsV8 = relations.data.data;
+}
+
+async function migrateRelationsData(context) {
+
+  const relationsV9 = context.relationsV8
+    .filter((relation) => {
+      return (
+        (relation.collection_many.startsWith("directus_") &&
+          relation.collection_one.startsWith("directus_")) === false
+      );
+    })
+    .map((relation) => ({
+      meta: {
+        many_collection: relation.collection_many,
+        many_field: relation.field_many,
+        one_collection: relation.collection_one,
+        one_field: relation.field_one,
+        junction_field: relation.junction_field,
+      },
+      field: relation.field_many,
+      collection: relation.collection_many,
+      related_collection: relation.collection_one,
+      schema: null,
+    }));
+
+  const systemFields = context.collections
+    .map((collection) =>
+      Object.values(collection.fields)
+        .filter((details) => {
+          return details.type === "file" || details.type.startsWith("user");
+        })
+        .map((field) => ({
+          meta: {
+            many_field: field.field,
+            many_collection: collection.collection,
+            one_collection:
+              field.type === "file" ? "directus_files" : "directus_users",
+          },
+          field: field.field,
+          collection: collection.collection,
+          related_collection:  field.type === "file" ? "directus_files" : "directus_users",
+          schema: null,
+        }))
+    )
+    .flat();
+
+  for (const relation of [...relationsV9, ...systemFields]) {
+    await apiV9.post("/relations", relation);
+  }
+
+  context.relations = [...relationsV9, ...systemFields];
+}