From 56b95abcf1c60b87b8107e5748e249c1a7742c6d Mon Sep 17 00:00:00 2001 From: Carlos Cabanero Date: Tue, 15 Mar 2022 20:23:00 -0400 Subject: [PATCH] FileProvider - Added Modify Item and various changes - ItemChanged implementation. - Proper signaling for enumerating changes. - Enumeration for files and directories. - Fixed issue with URL location in container when obtaining reference. - Improved verbosity --- .../FileProviderEnumerator.swift | 75 +++++++---- BlinkFileProvider/FileProviderExtension.swift | 126 ++++++++++++++---- .../Models/BlinkItemReference.swift | 30 ++++- .../Models/FileTranslatorCache.swift | 42 ++++-- 4 files changed, 204 insertions(+), 69 deletions(-) diff --git a/BlinkFileProvider/FileProviderEnumerator.swift b/BlinkFileProvider/FileProviderEnumerator.swift index 6ec2104a4..7656c6758 100644 --- a/BlinkFileProvider/FileProviderEnumerator.swift +++ b/BlinkFileProvider/FileProviderEnumerator.swift @@ -109,9 +109,9 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { .flatMap { // 2. Stat both local and remote files. // For remote, if the file is a link, then stat to know the real attributes - Publishers.Zip($0.directoryFilesAndAttributesResolvingLinks(), + Publishers.Zip($0.isDirectory ? $0.directoryFilesAndAttributesResolvingLinks() : AnyPublisher($0.stat().collect()), Local().walkTo(self.identifier.url.path) - .flatMap { $0.directoryFilesAndAttributes() } + .flatMap { $0.isDirectory ? $0.directoryFilesAndAttributes() : AnyPublisher($0.stat().collect()) } .catch { _ in AnyPublisher.just([]) }) } .map { (remoteFilesAttributes, localFilesAttributes) -> [BlinkItemReference] in @@ -153,24 +153,40 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { }).store(in: &cancellableBag) } -// func enumerateChanges(for observer: NSFileProviderChangeObserver, from anchor: NSFileProviderSyncAnchor) { -// /* TODO: -// - query the server for updates since the passed-in sync anchor -// -// If this is an enumerator for the active set: -// - note the changes in your local database -// -// - inform the observer about item deletions and updates (modifications + insertions) -// - inform the observer when you have finished enumerating up to a subsequent sync anchor -// */ -// // Schedule changes -// -// print("\(self.identifier.path) - Enumerating changes at \(currentAnchor) anchor") -// let data = "\(currentAnchor)".data(using: .utf8) -// observer.finishEnumeratingChanges(upTo: NSFileProviderSyncAnchor(data!), moreComing: false) -// -// } -// + func enumerateChanges(for observer: NSFileProviderChangeObserver, from anchor: NSFileProviderSyncAnchor) { + /* TODO: + - query the server for updates since the passed-in sync anchor + + If this is an enumerator for the active set: + - note the changes in your local database + + - inform the observer about item deletions and updates (modifications + insertions) + - inform the observer when you have finished enumerating up to a subsequent sync anchor + */ + // Schedule changes + + let anchor = UInt(String(data: anchor.rawValue, encoding: .utf8)!)! + self.log.info("Enumerating changes at \(anchor) anchor") + + guard let ref = FileTranslatorCache.reference(identifier: self.identifier) else { + observer.finishEnumeratingWithError("Op not supported") + return + } + + if let updatedItems = FileTranslatorCache.updatedItems(container: self.identifier, since: anchor) { + // Atm only update changes, no deletion as we don't provide tombstone values. + self.log.info("\(updatedItems.count) items updated.") + observer.didUpdate(updatedItems) + } else if anchor < ref.syncAnchor { + observer.didUpdate([ref]) + } + + let newAnchor = ref.syncAnchor + let data = "\(newAnchor)".data(using: .utf8) + + observer.finishEnumeratingChanges(upTo: NSFileProviderSyncAnchor(data!), moreComing: false) + } + /** Request the current sync anchor. @@ -195,12 +211,15 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { reasons, but are really required. System performance will be severely degraded if they are not implemented. */ -// func currentSyncAnchor(completionHandler: @escaping (NSFileProviderSyncAnchor?) -> Void) { -// -// // todo -// print("\(self.identifier.path) - Requested \(currentAnchor) anchor") -// -// let data = "\(currentAnchor)".data(using: .utf8) -// completionHandler(NSFileProviderSyncAnchor(data!)) -// } + func currentSyncAnchor(completionHandler: @escaping (NSFileProviderSyncAnchor?) -> Void) { + + guard let ref = FileTranslatorCache.reference(identifier: self.identifier) else { + completionHandler(nil) + return + } + self.log.info("Requested anchor \(ref.syncAnchor)") + + let data = "\(ref.syncAnchor)".data(using: .utf8) + completionHandler(NSFileProviderSyncAnchor(data!)) + } } diff --git a/BlinkFileProvider/FileProviderExtension.swift b/BlinkFileProvider/FileProviderExtension.swift index a28eaa232..e9e0e4723 100644 --- a/BlinkFileProvider/FileProviderExtension.swift +++ b/BlinkFileProvider/FileProviderExtension.swift @@ -45,7 +45,7 @@ class FileProviderExtension: NSFileProviderExtension { var fileManager = FileManager() var cancellableBag: Set = [] let copyArguments = CopyArguments(inplace: true, - preserve: [.permissions], + preserve: [.permissions, .timestamp], checkTimes: true) override init() { super.init() @@ -84,13 +84,16 @@ class FileProviderExtension: NSFileProviderExtension { // MARK: - BlinkItem Entry : DB-GET query (using uniq NSFileProviderItemIdentifier ID) override func item(for identifier: NSFileProviderItemIdentifier) throws -> NSFileProviderItem { - print("ITEM \(identifier.rawValue) REQUESTED") - + let log = BlinkLogger("itemFor") + log.info("\(identifier)") + var queryableIdentifier: BlinkItemIdentifier! if identifier == .rootContainer { guard let encodedRootPath = domain?.pathRelativeToDocumentStorage else { - throw NSFileProviderError(.noSuchItem) + let error = NSFileProviderError(.noSuchItem) + log.error("\(error)") + throw error } queryableIdentifier = BlinkItemIdentifier(encodedRootPath) } else { @@ -98,13 +101,13 @@ class FileProviderExtension: NSFileProviderExtension { } guard let reference = FileTranslatorCache.reference(identifier: queryableIdentifier) else { - if identifier == .rootContainer { - let attributes = try? fileManager.attributesOfItem(atPath: queryableIdentifier.url.path) - // Move operation requests root without enumarating. Return domain root with local attribtues - // TODO: Store in FileTranslatorCache? - return BlinkItemReference(queryableIdentifier, local: attributes) - } - print("ITEM \(queryableIdentifier.path) REQUESTED with ERROR") + if identifier == .rootContainer { + let attributes = try? fileManager.attributesOfItem(atPath: queryableIdentifier.url.path) + // Move operation requests root without enumarating. Return domain root with local attribtues + // TODO: Store in FileTranslatorCache? + return BlinkItemReference(queryableIdentifier, local: attributes) + } + log.error("No reference found for ITEM \(queryableIdentifier.path)") throw NSError.fileProviderErrorForNonExistentItem(withIdentifier: identifier) } @@ -177,15 +180,23 @@ class FileProviderExtension: NSFileProviderExtension { override func startProvidingItem(at url: URL, completionHandler: @escaping ((_ error: Error?) -> Void)) { // 1 - From URL we get the identifier. let log = BlinkLogger("startProvidingItem") + log.info("\(url).path") + //let blinkIdentifier = BlinkItemIdentifier(url: url) guard let blinkItemReference = FileTranslatorCache.reference(url: url) else { //guard let blinkItemReference = FileTranslatorCache.reference(identifier: blinkIdentifier) else { // TODO Proper error types (NSError) - completionHandler("Does not have a reference to copy") + log.error("No reference found") + completionHandler(NSFileProviderError(.noSuchItem)) return } - - log.info("\(blinkItemReference.path) - start") + + guard !blinkItemReference.isDownloaded else { + log.info("\(blinkItemReference.path) - current item up to date") + completionHandler(nil) + return + } + // 2 local translator let destTranslator = Local().cloneWalkTo(url.deletingLastPathComponent().path) @@ -203,14 +214,16 @@ class FileProviderExtension: NSFileProviderExtension { log.info("\(blinkItemReference.path) - completed") blinkItemReference.downloadCompleted(nil) completionHandler(nil) - NSFileProviderManager.default.signalEnumerator(for: blinkItemReference.itemIdentifier, completionHandler: { _ in }) + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) case .failure(let error): + log.error("\(error)") completionHandler(NSFileProviderError.operationError(dueTo: error)) - NSFileProviderManager.default.signalEnumerator(for: blinkItemReference.itemIdentifier, completionHandler: { _ in }) + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) } }, receiveValue: { _ in }) blinkItemReference.downloadStarted(downloadTask) + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) } override func stopProvidingItem(at url: URL) { @@ -270,6 +283,7 @@ class FileProviderExtension: NSFileProviderExtension { // Copy only Regular files, do not support directories yet. if attributes[.type] as! FileAttributeType != .typeRegular { + log.error("Directories not supported for this operation") completionHandler(nil, NSFileProviderError(.noSuchItem)) return } @@ -306,6 +320,7 @@ class FileProviderExtension: NSFileProviderExtension { blinkItemReference.uploadCompleted(error) completionHandler(blinkItemReference, NSFileProviderError.operationError(dueTo: error)) + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) return } @@ -314,23 +329,75 @@ class FileProviderExtension: NSFileProviderExtension { // the state of the file would not change. log.info("Upload completed \(localFileURLPath)") completionHandler(blinkItemReference, nil) + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) } receiveValue: { _ in } blinkItemReference.uploadStarted(c) + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) } override func itemChanged(at url: URL) { - BlinkLogger("Unsupported itemChanged").debug("\(url.path)") - // Called at some point after the file has changed; the provider may then trigger an upload + let log = BlinkLogger("itemChanged") + log.info("\(url.path)") + + guard let blinkItemReference = FileTranslatorCache.reference(url: url) else { + log.error("Could not find reference to item") + return + } - /* TODO: - - mark file at as needing an update in the model - - if there are existing NSURLSessionTasks uploading this file, cancel them - - create a fresh background NSURLSessionTask and schedule it to upload the current modifications - - register the NSURLSessionTask with NSFileProviderManager to provide progress updates - */ + // - mark file at as needing an update in the model + // Update the model + var attributes: FileAttributes! + do { + attributes = try fileManager.attributesOfItem(atPath: url.path) + attributes[.name] = url.lastPathComponent + } catch { + log.error("Could not fetch attributes of item - \(error)") + return + } + blinkItemReference.updateAttributes(remote: blinkItemReference.remote!, local: attributes) + + // - if there are existing NSURLSessionTasks uploading this file, cancel them + // Cancel an upload if there is a reference to it. + blinkItemReference.uploadingTask?.cancel() + + // - create a fresh background NSURLSessionTask and schedule it to upload the current modifications + // 1. Translator for local target path + let localFileURLPath = url.path + let srcTranslator = Local().cloneWalkTo(localFileURLPath) + // 2. Translator for remote file path + let itemIdentifier = blinkItemReference.itemIdentifier + let destTranslator = FileTranslatorCache.translator(for: BlinkItemIdentifier(itemIdentifier)) + .flatMap { $0.cloneWalkTo(BlinkItemIdentifier(blinkItemReference.parentItemIdentifier).path) } + + // 3. Upload + let c = destTranslator.flatMap { remotePathTranslator in + return srcTranslator.flatMap{ localFileTranslator -> CopyProgressInfoPublisher in + // 3. Start copy + return remotePathTranslator.copy(from: [localFileTranslator], + args: self.copyArguments) + } + }.sink { completion in + // 4. Update reference and notify + if case let .failure(error) = completion { + log.error("Upload failed \(localFileURLPath)- \(error)") + blinkItemReference.uploadCompleted(error) + + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) + return + } + + blinkItemReference.uploadCompleted(nil) + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) + + log.info("Upload completed \(localFileURLPath)") + } receiveValue: { _ in } + + blinkItemReference.uploadStarted(c) + + self.signalEnumerator(for: blinkItemReference.parentItemIdentifier) } override func createDirectory(withName directoryName: String, @@ -553,6 +620,17 @@ class FileProviderExtension: NSFileProviderExtension { } } + private func signalEnumerator(for container: NSFileProviderItemIdentifier) { + guard let domain = self.domain, + let fpm = NSFileProviderManager(for: domain) else { + return + } + + fpm.signalEnumerator(for: container, completionHandler: { error in + BlinkLogger("signalEnumerator").info("Enumerator Signaled with \(error ?? "no error")") + }) + } + deinit { print("OOOOUUUTTTTT!!!!!") } diff --git a/BlinkFileProvider/Models/BlinkItemReference.swift b/BlinkFileProvider/Models/BlinkItemReference.swift index 15c673edf..3eb7ff092 100644 --- a/BlinkFileProvider/Models/BlinkItemReference.swift +++ b/BlinkFileProvider/Models/BlinkItemReference.swift @@ -55,6 +55,8 @@ final class BlinkItemReference: NSObject { var isUploaded: Bool = false var uploadingError: Error? = nil + var syncAnchor: UInt = 0 + // MARK: - Enumerator Entry Point: // Requires attributes. If you only have the Identifier, you need to go to the DB. // Identifier format /path/to/more/components/filename @@ -76,6 +78,14 @@ final class BlinkItemReference: NSObject { self.local = local } evaluate() + updateSyncAnchor() + } + + // Sync anchor of the container is increased when an item inside it changes. + // The sync anchor for an item itself is the one that updated the parent. + private func updateSyncAnchor() { + self.parentItem?.syncAnchor += 1 + self.syncAnchor = self.parentItem?.syncAnchor ?? self.syncAnchor + 1 } private func evaluate() { @@ -93,12 +103,15 @@ final class BlinkItemReference: NSObject { return } - if remoteModified > localModified { + // Floor modified times as on some platforms it is a dobule with decimals + let epochRemote = floor(remoteModified.timeIntervalSince1970) + let epochLocal = floor(localModified.timeIntervalSince1970) + if epochRemote > epochLocal { primary = remote! replica = local isDownloaded = false isUploaded = true - } else if remoteModified == localModified { + } else if epochRemote == epochLocal { primary = remote! replica = local isDownloaded = true @@ -112,6 +125,11 @@ final class BlinkItemReference: NSObject { } } + var parentItem: BlinkItemReference? { + FileTranslatorCache + .reference(identifier: BlinkItemIdentifier(self.parentItemIdentifier)) + } + var path: String { identifier.path } @@ -145,6 +163,7 @@ final class BlinkItemReference: NSObject { func downloadStarted(_ c: AnyCancellable) { downloadingTask = c downloadingError = nil + updateSyncAnchor() evaluate() } @@ -156,13 +175,15 @@ final class BlinkItemReference: NSObject { } local = remote - evaluate() downloadingTask = nil + updateSyncAnchor() + evaluate() } func uploadStarted(_ c: AnyCancellable) { uploadingTask = c uploadingError = nil + updateSyncAnchor() } func uploadCompleted(_ error: Error?) { @@ -173,8 +194,9 @@ final class BlinkItemReference: NSObject { } remote = local - evaluate() uploadingTask = nil + updateSyncAnchor() + evaluate() } } diff --git a/BlinkFileProvider/Models/FileTranslatorCache.swift b/BlinkFileProvider/Models/FileTranslatorCache.swift index fc3076a27..77d1b301c 100644 --- a/BlinkFileProvider/Models/FileTranslatorCache.swift +++ b/BlinkFileProvider/Models/FileTranslatorCache.swift @@ -65,7 +65,7 @@ final class FileTranslatorCache { static let shared = FileTranslatorCache() private var translators: [String: TranslatorControl] = [:] private var references: [String: BlinkItemReference] = [:] - private var fileList: [String: [BlinkItemReference]] = [:] + private var fileList = [String: [BlinkItemReference]]() private var backgroundThread: Thread? = nil private var backgroundRunLoop: RunLoop = RunLoop.current @@ -131,7 +131,16 @@ final class FileTranslatorCache { static func store(reference: BlinkItemReference) { print("storing File BlinkItemReference : \(reference.itemIdentifier.rawValue)") shared.references[reference.itemIdentifier.rawValue] = reference + if reference.itemIdentifier != .rootContainer { + if var list = shared.fileList[reference.parentItemIdentifier.rawValue] { + list.append(reference) + shared.fileList[reference.parentItemIdentifier.rawValue] = list + } else { + shared.fileList[reference.parentItemIdentifier.rawValue] = [reference] + } + } } + static func remove(reference: BlinkItemReference) { shared.references.removeValue(forKey: reference.itemIdentifier.rawValue) } @@ -142,20 +151,27 @@ final class FileTranslatorCache { } static func reference(url: URL) -> BlinkItemReference? { - let manager = NSFileProviderManager.default - let containerPath = manager.documentStorageURL.path - - // file://///filename - // file:////path/filename - // Remove containerPath, split and get encodedRootPath. - var encodedPath = url.path - encodedPath.removeFirst(containerPath.count) - if encodedPath.hasPrefix("/") { - encodedPath.removeFirst() + // containerPath may not be the same when accessing for different app. It may have a /private prefix. + // To obtain the reference, we delete up to File Provider Storage. + // file:///File Provider Storage///filename + // file:///File Provider Storage//path/filename + let encodedPath = url.path + guard let range: Range = encodedPath.range(of: "File Provider Storage/") else { + return nil + } + + var cleanPath = encodedPath[range.upperBound...] + if cleanPath.hasPrefix("/") { + cleanPath.removeFirst() } - // ///filename - return shared.references[encodedPath] + return shared.references[String(cleanPath)] + } + + static func updatedItems(container: BlinkItemIdentifier, since anchor: UInt) -> [BlinkItemReference]? { + shared.fileList[container.itemIdentifier.rawValue]?.filter { + anchor < $0.syncAnchor + } } }