feat(upload): upload files instead of using dataURL

dataURL's performance is bad

.env.example

@@ -5,4 +5,5 @@ OAUTH_GOOGLE_CLIENT_ID=
 OAUTH_GOOGLE_CLIENT_SECRET=
 GLOBAL_AGENT_HTTP_PROXY=
 SMTP_MAIL=
-MAIL_URL=
+MAIL_URL=
+UPLOAD_ABSOLUTE_PATH=

.meteor/packages

@@ -29,3 +29,4 @@ [email protected]
 [email protected]
 [email protected]
 markboard:scss-modules
+ostrio:files

.meteor/versions

@@ -62,6 +62,8 @@ [email protected]
 [email protected]
 [email protected]
 [email protected]
+ostrio:[email protected]
+ostrio:[email protected]
 [email protected]
 [email protected]
 [email protected]

imports/excalidraw/components/BoardTitleInput.tsx

@@ -15,12 +15,8 @@ export const BoardTitleInput = () => {
     Services.get('board').changeBoardTitle(boardId, title)
   }, 200)
 
-  const currentBoard = privateBoards.find(({ id }) => id === getCurrentBoardId())
-  console.log('currentBoard2', currentBoard)
-
   useEffect(() => {
     const currentBoard = privateBoards.find(({ id }) => id === getCurrentBoardId())
-    console.log('currentBoard', currentBoard)
     if (currentBoard != null) {
       const element = $input.current!
       element.value = currentBoard.title ?? 'Untitled'

imports/excalidraw/components/ExcalidrawCore.tsx

@@ -273,6 +273,7 @@ import { atom } from 'jotai'
 import { Fonts } from '../scene/Fonts'
 import { actionPaste } from '../actions/actionClipboard'
 import { attachDebugLabel } from '/imports/store'
+import { uploadFile } from '/imports/services/client/files'
 
 export const isMenuOpenAtom = attachDebugLabel(atom(false), 'isMenuOpenAtom')
 export const isDropdownOpenAtom = attachDebugLabel(atom(false), 'isDropdownOpenAtom')
@@ -4869,8 +4870,6 @@ export class ExcalidrawCore extends React.Component<AppProps, AppState> {
       this.setImagePreviewCursor(resizedFile || imageFile)
     }
 
-    const dataURL = this.files[fileId]?.dataURL || (await getDataURL(imageFile))
-
     const imageElement = mutateElement(
       _imageElement,
       {
@@ -4886,7 +4885,7 @@ export class ExcalidrawCore extends React.Component<AppProps, AppState> {
           [fileId]: {
             mimeType,
             id: fileId,
-            dataURL,
+            dataURL: (await uploadFile(imageFile)) as DataURL,
             created: Date.now(),
             lastRetrieved: Date.now()
           }

imports/models/Collections.ts

@@ -1,11 +1,28 @@
 import { Mongo } from 'meteor/mongo'
 import { Document } from 'bson'
+import { FilesCollection } from 'meteor/ostrio:files'
+import { Meteor } from 'meteor/meteor'
+
+if (Meteor.isServer && !process.env.UPLOAD_ABSOLUTE_PATH) {
+  throw new Error(
+    'Envrionment variable UPLOAD_ABSOLUTE_PATH is not set, ' +
+      'please set it to the absolute path of the upload folder'
+  )
+}
 
 class CollectionsClass {
   private _collections = {
     boards: new Mongo.Collection('boards'),
     logs: new Mongo.Collection('logs'),
-    appState: new Mongo.Collection('appState')
+    appState: new Mongo.Collection('appState'),
+    files: new FilesCollection({
+      storagePath: process.env.UPLOAD_ABSOLUTE_PATH,
+      collectionName: 'files',
+      allowClientCode: false,
+      onBeforeUpload(_file: File) {
+        return true
+      }
+    })
   } as const
 
   public getCollectionByName(name: string) {

imports/services/FilesService.ts

@@ -0,0 +1,30 @@
+import { Collections } from '../models/Collections'
+import { BaseService } from './BaseService'
+import { WebApp } from 'meteor/webapp'
+import fs from 'fs'
+
+export class FilesService extends BaseService {
+  constructor() {
+    super('files')
+  }
+
+  public startup(): void {
+    const FilesCollection = Collections.getCollectionByName('files')
+    WebApp.rawConnectHandlers.use((req, res, next) => {
+      if (req.url?.startsWith('/files/')) {
+        const matches = req.url!.match(/\/files\/([^/]+)/)
+        if (matches != null) {
+          const id = matches[1]
+          const cursor = FilesCollection.findOne(id)
+          if (cursor) {
+            const readStream = fs.createReadStream(cursor.get('path'))
+            return readStream.pipe(res)
+          }
+        }
+        res.writeHead(404)
+        return res.end()
+      }
+      return next()
+    })
+  }
+}

imports/services/ServiceManagerServer.ts

@@ -5,6 +5,7 @@ import { ExcalidrawSyncService } from './ExcalidrawSyncService'
 import { StaticAssetsService } from './StaticAssetsService'
 import { AppService } from './AppService'
 import { ServerServices } from './types'
+import { FilesService } from './FilesService'
 
 class ServiceManagerServerClass {
   private _services = {} as ServerServices
@@ -15,6 +16,7 @@ class ServiceManagerServerClass {
     this._services['excalidrawSync'] = new ExcalidrawSyncService()
     this._services['staticAssets'] = new StaticAssetsService()
     this._services['board'] = new BoardService()
+    this._services['files'] = new FilesService()
   }
 
   public getService<T extends keyof ServerServices>(service: T): ServerServices[T] {

imports/services/client/files.ts

@@ -0,0 +1,26 @@
+import { Collections } from '/imports/models/Collections'
+import { Document } from 'bson'
+
+export function uploadFile(file: File): Promise<string> {
+  const FilesCollection = Collections.getCollectionByName('files') as any
+  return new Promise((resolve, reject) => {
+    const upload = FilesCollection.insert(
+      {
+        file,
+        chunkSize: 'dynamic',
+        meta: {},
+        transport: 'http'
+      },
+      false
+    )
+    upload.on('end', (error: Error, file: Document) => {
+      if (error) {
+        console.error('upload fail', error)
+        reject(error)
+      } else {
+        resolve(`/files/${file._id}`)
+      }
+    })
+    upload.start()
+  })
+}

imports/services/types.ts

@@ -3,6 +3,7 @@ import type { BoardService } from './BoardService'
 import type { ExcalidrawSyncService } from './ExcalidrawSyncService'
 import type { StaticAssetsService } from './StaticAssetsService'
 import type { AppService } from './AppService'
+import type { FilesService } from './FilesService'
 
 type OmitPrivateProperties<T> = Omit<T, '_startup' | 'startup' | 'serviceName'>
 
@@ -20,6 +21,7 @@ export interface IServices {
   excalidrawSync: ExcalidrawSyncService
   staticAssets: StaticAssetsService
   board: BoardService
+  files: FilesService
 }
 
 export type ServerServices = {

imports/subscriptions/BoardsSubscriber.ts

@@ -52,20 +52,16 @@ class BoardsSubscriberClass {
 
   private _subscribe() {
     Meteor.subscribe(Collections.names.boards)
-    console.log('observeChanges')
     const BoardCollection = Collections.getCollectionByName('boards')
     const onBoardsChanged = this._onBoardsChanged.bind(this)
     BoardCollection.find({}).observeChanges({
-      added(id, fields) {
-        console.log('boards added:', id, fields)
+      added(_id, _fields) {
         onBoardsChanged()
       },
-      changed(id, fields) {
-        console.log('boards changed:', id, fields)
+      changed(_id, _fields) {
         onBoardsChanged()
       },
-      removed(id) {
-        console.log('boards removed:', id)
+      removed(_id) {
         onBoardsChanged()
       }
     })

types/ostrio:files.d.ts

@@ -0,0 +1,4 @@
+declare module 'meteor/ostrio:files' {
+  export const FilesCollection: any
+  export const GridFS: any
+}