User configurable media auto-fetching

.gitmodules

@@ -36,4 +36,4 @@
 	url = [email protected]:ChatSecure/JSQMessagesViewController.git
 [submodule "Carthage/Checkouts/HTMLReader"]
 	path = Carthage/Checkouts/HTMLReader
-	url = https://github.com/nolanw/HTMLReader.git
+	url = https://github.com/ChatSecure/HTMLReader.git

Cartfile

@@ -2,4 +2,5 @@
 github "ChatSecure/SignalProtocol-ObjC" "241f439"
 github "ChatSecure/SignalProtocolC" "4335dfb"
 github "ChatSecure/Mantle" "2.1.0_headerfix"
-github "nolanw/HTMLReader" "v2.0.5"
+# github "nolanw/HTMLReader" "v2.0.5"
+github "ChatSecure/HTMLReader" "xcode9-fix"

Cartfile.resolved

@@ -1,4 +1,4 @@
+github "ChatSecure/HTMLReader" "808c2ebedc8b044eefc7a46c2dfca593244922e1"
 github "ChatSecure/Mantle" "4c1a09cb0c0811956cd35262340e42b940971cbb"
 github "ChatSecure/SignalProtocol-ObjC" "241f439a80afd68d9199e73e95f1350ade893b53"
 github "ChatSecure/SignalProtocolC" "4335dfb70f6eb036c409cea59ebc778f4d734c16"
-github "nolanw/HTMLReader" "v2.0.5"

ChatSecure/Classes/Controllers/FileTransferManager.swift

@@ -372,7 +372,7 @@ public class FileTransferManager: NSObject, OTRServerCapabilitiesDelegate {
 extension FileTransferManager {
 
     /** creates downloadmessages and then downloads if needed. parent message should already be saved! @warn Do not call from within an existing db transaction! */
-    public func createAndDownloadItemsIfNeeded(message: OTRDownloadMessageProtocol, readConnection: YapDatabaseConnection) {
+    public func createAndDownloadItemsIfNeeded(message: OTRMessageProtocol, readConnection: YapDatabaseConnection) {
         if message.messageMediaItemKey != nil || message.messageText?.characters.count == 0 || message.downloadableURLs.count == 0 {
             DDLogVerbose("Download of message not needed \(message.messageKey)")
             return
@@ -494,6 +494,19 @@ public extension OTRBaseMessage {
 
 // MARK: - Extensions
 
+fileprivate extension XMPPMessage {
+    /** XEP-0066: Out of Band Data jabber:x:oob */
+    var outOfBandURL: URL? {
+        guard let oob = elements(forXmlns: "jabber:x:oob").first as? XMLElement,
+            let urlElement = oob.elements(forName: "url").first,
+            let urlString = urlElement.stringValue else {
+                return nil
+        }
+        let url = URL(string: urlString)
+        return url
+    }
+}
+
 fileprivate struct HTTPServer {
     /// service jid for upload service
     let jid: XMPPJID
@@ -549,11 +562,6 @@ enum URLScheme: String {
     static let downloadableSchemes: [URLScheme] = [.https, .aesgcm]
 }
 
-enum MimeTypes: String {
-    case jpeg = "image/jpeg"
-    case png = "image/png"
-}
-
 extension URL {
 
     /** URL scheme matches aesgcm:// */

ChatSecure/Classes/Controllers/MessageQueueHandler.swift

@@ -538,7 +538,7 @@ extension MessageQueueHandler {
 
 
 
-        signalCoordinator.encryptAndSendMessage(text, buddyYapKey: message.buddyUniqueId, messageId: message.messageId, completion: { [weak self] (success, error) in
+        signalCoordinator.encryptAndSendMessage(message, buddyYapKey: message.buddyUniqueId, messageId: message.messageId, completion: { [weak self] (success, error) in
             guard let strongSelf = self else {
                 return
             }
@@ -547,7 +547,7 @@ extension MessageQueueHandler {
                 //Something went wrong getting ready to send the message
                 //Save error object to message
                 strongSelf.databaseConnection.readWrite({ (transaction) in
-                    guard let message = OTROutgoingMessage.fetchObject(withUniqueID: message.uniqueId, transaction: transaction)?.copy() as? OTROutgoingMessage else {
+                    guard let message = message.refetch(with: transaction) else {
                         return
                     }
                     message.error = error

ChatSecure/Classes/Controllers/OTRDatabaseView.m

@@ -208,21 +208,15 @@ + (BOOL)registerFilteredChatViewWithDatabase:(YapDatabase *)database {
     }
     YapDatabaseViewOptions *options = [[YapDatabaseViewOptions alloc] init];
     YapDatabaseViewFiltering *filtering = [YapDatabaseViewFiltering withObjectBlock:^BOOL(YapDatabaseReadTransaction * _Nonnull transaction, NSString * _Nonnull group, NSString * _Nonnull collection, NSString * _Nonnull key, id  _Nonnull object) {
-        if ([object conformsToProtocol:@protocol(OTRDownloadMessageProtocol)]) {
-            id<OTRDownloadMessageProtocol> message = object;
+        if ([object conformsToProtocol:@protocol(OTRMessageProtocol)]) {
+            id<OTRMessageProtocol> message = object;
             // Filter out messages that are just URLs and have downloads
             if (!message.messageText &&
                 !message.messageMediaItemKey &&
                 [message hasExistingDownloadsWithTransaction:transaction]) {
                 return NO;
             }
         }
-//        if ([object isKindOfClass:[OTRDownloadMessage class]]) {
-//            OTRDownloadMessage *download = object;
-//            if (!download.messageMediaItemKey) {
-//                return NO;
-//            }
-//        }
 
         return YES;
     }];

ChatSecure/Classes/Controllers/OTROMEMOSignalCoordinator.swift

@@ -183,7 +183,7 @@ import SignalProtocolObjC
      - parameter messageId: The preffered XMPP element Id to be used.
      - parameter completion: The completion block is called after all the necessary omemo preperation has completed and sendKeyData:iv:toJID:payload:elementId: is invoked
      */
-    open func encryptAndSendMessage(_ messageBody:String, buddyYapKey:String, messageId:String?, completion:@escaping (Bool,NSError?) -> Void) {
+    open func encryptAndSendMessage(_ message: OTROutgoingMessage, buddyYapKey:String, messageId:String?, completion:@escaping (Bool,NSError?) -> Void) {
         // Gather bundles for buddy and account here
         let group = DispatchGroup()
 
@@ -207,7 +207,8 @@ import SignalProtocolObjC
                 bud = OTRBuddy.fetchObject(withUniqueID: buddyYapKey, transaction: transaction)
             }
 
-            guard let ivData = OTRSignalEncryptionHelper.generateIV(), let keyData = OTRSignalEncryptionHelper.generateSymmetricKey(), let messageBodyData = messageBody.data(using: String.Encoding.utf8) , let buddy = bud else {
+            guard let messageBody = message.text,
+                let ivData = OTRSignalEncryptionHelper.generateIV(), let keyData = OTRSignalEncryptionHelper.generateSymmetricKey(), let messageBodyData = messageBody.data(using: String.Encoding.utf8) , let buddy = bud else {
                 return
             }
             do {

ChatSecure/Classes/Controllers/XMPP/OTRXMPPManager.m

@@ -705,6 +705,30 @@ - (void)changePassword:(NSString *)newPassword completion:(void (^)(BOOL,NSError
 #pragma mark XMPPStream Delegate
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
+- (XMPPMessage *)xmppStream:(XMPPStream *)sender willSendMessage:(XMPPMessage *)message {
+    NSString *body = [message body];
+    if (body.length && ![body containsString:@" "]) {
+        NSURL *url = [NSURL URLWithString:body];
+        NSString *extension = url.pathExtension;
+        NSString *scheme = url.scheme;
+        if (scheme.length &&
+            extension.length &&
+            ([scheme isEqualToString:@"https"])) {
+            // this means we're probably sending a file so mark it as OOB data
+            /*
+             <x xmlns="jabber:x:oob">
+               <url>https://xmpp.example.com/upload/6c44c22c-70c6-4375-8b2d-1992a0f9dc18/8CleEoqhTjiMcny0PqoZyg.jpg</url>
+             </x>
+             */
+            NSXMLElement *oob = [NSXMLElement elementWithName:@"x" xmlns:@"jabber:x:oob"];
+            NSXMLElement *urlElement = [NSXMLElement elementWithName:@"url" stringValue:body];
+            [oob addChild:urlElement];
+            [message addChild:oob];
+        }
+    }
+    return message;
+}
+
 - (void)xmppStreamDidChangeMyJID:(XMPPStream *)stream
 {
     if (![[stream.myJID bare] isEqualToString:self.account.username] || ![[stream.myJID resource] isEqualToString:self.account.resource])

ChatSecure/Classes/Model/Yap Storage/Accounts/OTRAccount.h

@@ -43,6 +43,8 @@ extern NSString *const OTRXMPPTorImageName;
 @property (nonatomic) BOOL autologin;
 
 @property (nonatomic, readonly) BOOL isArchived;
+/** Whether or not user would like to auto fetch media messages */
+@property (nonatomic, readwrite) BOOL disableAutomaticURLFetching;
 
 /**
  * Setting this value does a comparison of against the previously value

ChatSecure/Classes/Model/Yap Storage/Accounts/OTRXMPPTorAccount.m

@@ -24,6 +24,10 @@ - (Class)protocolClass{
     return [OTRXMPPTorManager class];
 }
 
+- (BOOL) disableAutomaticURLFetching {
+    return YES;
+}
+
 #pragma - mark Class Methods
 
 + (NSString *)collection

ChatSecure/Classes/Model/Yap Storage/OTRBaseMessage.h

@@ -15,7 +15,17 @@
 NS_ASSUME_NONNULL_BEGIN
 @class OTRDownloadMessage;
 
-@protocol OTRMessageProtocol <OTRYapDatabaseObjectProtocol>
+@protocol OTRDownloadMessageProtocol <NSObject>
+@required
+/** Returns an unsaved array of downloadable URLs. */
+- (NSArray<OTRDownloadMessage*>*) downloads;
+/**  If available, existing instances will be returned. */
+- (NSArray<OTRDownloadMessage*>*) existingDownloadsWithTransaction:(YapDatabaseReadTransaction*)transaction;
+/** Checks if edge count > 0 */
+- (BOOL) hasExistingDownloadsWithTransaction:(YapDatabaseReadTransaction*)transaction;
+@end
+
+@protocol OTRMessageProtocol <OTRYapDatabaseObjectProtocol, OTRDownloadMessageProtocol>
 @required
 @property (nonatomic, readonly) NSString *messageKey;
 @property (nonatomic, readonly) NSString *messageCollection;
@@ -32,17 +42,8 @@ NS_ASSUME_NONNULL_BEGIN
 - (nullable id<OTRThreadOwner>)threadOwnerWithTransaction:(nonnull YapDatabaseReadTransaction *)transaction;
 @end
 
-@protocol OTRDownloadMessageProtocol <OTRMessageProtocol>
-@required
-/** Returns an unsaved array of downloadable URLs. */
-- (NSArray<OTRDownloadMessage*>*) downloads;
-/**  If available, existing instances will be returned. */
-- (NSArray<OTRDownloadMessage*>*) existingDownloadsWithTransaction:(YapDatabaseReadTransaction*)transaction;
-/** Checks if edge count > 0 */
-- (BOOL) hasExistingDownloadsWithTransaction:(YapDatabaseReadTransaction*)transaction;
-@end
 
-@interface OTRBaseMessage : OTRYapDatabaseObject <YapDatabaseRelationshipNode, OTRMessageProtocol, OTRDownloadMessageProtocol>
+@interface OTRBaseMessage : OTRYapDatabaseObject <YapDatabaseRelationshipNode, OTRMessageProtocol>
 
 /** The date the message is created for outgoing messages and the date it is received for incoming messages*/
 @property (nonatomic, strong, nonnull) NSDate *date;

ChatSecure/Classes/Model/Yap Storage/PushMessage.swift

@@ -28,6 +28,18 @@ open class PushMessage: OTRYapDatabaseObject {
 }
 
 extension PushMessage: OTRMessageProtocol {
+    public func downloads() -> [OTRDownloadMessage] {
+        return []
+    }
+
+    public func existingDownloads(with transaction: YapDatabaseReadTransaction) -> [OTRDownloadMessage] {
+        return []
+    }
+
+    public func hasExistingDownloads(with transaction: YapDatabaseReadTransaction) -> Bool {
+        return false
+    }
+
     public var messageKey: String {
         return self.uniqueId
     }

ChatSecure/Classes/View Controllers/Login View Controllers/OTRXLFormCreator.h

@@ -22,6 +22,8 @@ extern NSString *const kOTRXLFormPortTextFieldTag;
 extern NSString *const kOTRXLFormResourceTextFieldTag;
 extern NSString *const kOTRXLFormXMPPServerTag;
 extern NSString *const kOTRXLFormUseTorTag;
+extern NSString *const kOTRXLFormAutomaticURLFetchTag;
+
 
 @interface XLFormDescriptor (OTRAccount)
 

ChatSecure/Classes/View Controllers/Login View Controllers/OTRXLFormCreator.m

@@ -34,6 +34,8 @@
 NSString *const kOTRXLFormGenerateSecurePasswordTag               = @"kOTRXLFormGenerateSecurePasswordTag";
 
 NSString *const kOTRXLFormUseTorTag               = @"kOTRXLFormUseTorTag";
+NSString *const kOTRXLFormAutomaticURLFetchTag               = @"kOTRXLFormAutomaticURLFetchTag";
+
 
 @implementation XLFormDescriptor (OTRAccount)
 
@@ -56,6 +58,7 @@ + (instancetype) existingAccountFormWithAccount:(OTRAccount *)account
             XLFormRowDescriptor *torRow = [descriptor formRowWithTag:kOTRXLFormUseTorTag];
             torRow.hidden = @YES;
         }
+        [[descriptor formRowWithTag:kOTRXLFormAutomaticURLFetchTag] setValue:@(!xmppAccount.disableAutomaticURLFetching)];
     }
     if (account.accountType == OTRAccountTypeXMPPTor) {
         XLFormRowDescriptor *torRow = [descriptor formRowWithTag:kOTRXLFormUseTorTag];
@@ -64,6 +67,9 @@ + (instancetype) existingAccountFormWithAccount:(OTRAccount *)account
         XLFormRowDescriptor *autologin = [descriptor formRowWithTag:kOTRXLFormLoginAutomaticallySwitchTag];
         autologin.value = @NO;
         autologin.disabled = @YES;
+        XLFormRowDescriptor *autofetch = [descriptor formRowWithTag:kOTRXLFormAutomaticURLFetchTag];
+        autofetch.value = @NO;
+        autofetch.disabled = @YES;
     }
 
     return descriptor;
@@ -125,11 +131,17 @@ + (XLFormDescriptor *)formForAccountType:(OTRAccountType)accountType createAccou
         torSection.hidden = [NSString stringWithFormat:@"$%@==0", kOTRXLFormShowAdvancedTag];
         [torSection addFormRow:[self torRowDescriptorWithValue:NO]];
 
+        XLFormSectionDescriptor *otherSection = [XLFormSectionDescriptor formSectionWithTitle:OTHER_STRING()];
+        otherSection.footerTitle = AUTO_URL_FETCH_WARNING_STRING();
+        otherSection.hidden = [NSString stringWithFormat:@"$%@==0", kOTRXLFormShowAdvancedTag];
+        [otherSection addFormRow:[self autoFetchRowDescriptorWithValue:YES]];
+
         [descriptor addFormSection:basicSection];
         [descriptor addFormSection:serverSection];
         [descriptor addFormSection:showAdvancedSection];
         [descriptor addFormSection:accountSection];
         [descriptor addFormSection:torSection];
+        [descriptor addFormSection:otherSection];
     } else {
         descriptor = [XLFormDescriptor formDescriptorWithTitle:LOGIN_STRING()];
         XLFormSectionDescriptor *basicSection = [XLFormSectionDescriptor formSectionWithTitle:BASIC_STRING()];
@@ -151,6 +163,7 @@ + (XLFormDescriptor *)formForAccountType:(OTRAccountType)accountType createAccou
                 [advancedSection addFormRow:[self resourceRowDescriptorWithValue:[OTRXMPPAccount newResource]]];
 
                 [advancedSection addFormRow:[self torRowDescriptorWithValue:NO]];
+                [advancedSection addFormRow:[self autoFetchRowDescriptorWithValue:YES]];
 
                 break;
             }
@@ -162,6 +175,7 @@ + (XLFormDescriptor *)formForAccountType:(OTRAccountType)accountType createAccou
                 [basicSection addFormRow:[self loginAutomaticallyRowDescriptorWithValue:YES]];
 
                 [advancedSection addFormRow:[self resourceRowDescriptorWithValue:nil]];
+                [advancedSection addFormRow:[self autoFetchRowDescriptorWithValue:YES]];
 
                 break;
             }
@@ -248,6 +262,12 @@ + (XLFormRowDescriptor *)portRowDescriptorWithValue:(NSNumber *)value
     return portRowDescriptor;
 }
 
++ (XLFormRowDescriptor*) autoFetchRowDescriptorWithValue:(BOOL)value {
+    XLFormRowDescriptor *autoFetchRow = [XLFormRowDescriptor formRowDescriptorWithTag:kOTRXLFormAutomaticURLFetchTag rowType:XLFormRowDescriptorTypeBooleanSwitch title:AUTO_URL_FETCH_STRING()];
+    autoFetchRow.value = @(value);
+    return autoFetchRow;
+}
+
 + (XLFormRowDescriptor*) torRowDescriptorWithValue:(BOOL)value {
     XLFormRowDescriptor *torRow = [XLFormRowDescriptor formRowDescriptorWithTag:kOTRXLFormUseTorTag rowType:XLFormRowDescriptorTypeBooleanSwitch title:Enable_Tor_String()];
     torRow.value = @(value);

ChatSecure/Classes/View Controllers/OTRMessagesViewController.m

@@ -1194,11 +1194,17 @@ - (UICollectionViewCell *)collectionView:(JSQMessagesCollectionView *)collection
     // TODO: move this hack to download media items to
     // somewhere else
     __block OTRXMPPManager *xmpp = nil;
+    __block OTRXMPPAccount *account = nil;
     [self.readOnlyDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction * _Nonnull transaction) {
         xmpp = [self xmppManagerWithTransaction:transaction];
+        account = (OTRXMPPAccount*)[self accountWithTransaction:transaction];
     }];
-    if (xmpp && [message isKindOfClass:[OTRIncomingMessage class]]) {
-        [xmpp.fileTransferManager createAndDownloadItemsIfNeededWithMessage:(OTRIncomingMessage*)message readConnection:self.readOnlyDatabaseConnection];
+    if (xmpp && [message isKindOfClass:[OTRIncomingMessage class]] &&
+        [account isKindOfClass:[OTRXMPPAccount class]]) {
+        // Do not automatically download messages if disabled by user
+        if (!account.disableAutomaticURLFetching) {
+            [xmpp.fileTransferManager createAndDownloadItemsIfNeededWithMessage:(OTRIncomingMessage*)message readConnection:self.readOnlyDatabaseConnection];
+        }
     }
 
     UIColor *textColor = nil;
@@ -1213,10 +1219,6 @@ - (UICollectionViewCell *)collectionView:(JSQMessagesCollectionView *)collection
 
 	// Do not allow clickable links for Tor accounts to prevent information leakage
     // Could be better to move this information to the message object to not need to do a database read.
-    __block OTRAccount *account = nil;
-    [self.readOnlyDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction * _Nonnull transaction) {
-        account = [self accountWithTransaction:transaction];
-    }];
     if ([account isKindOfClass:[OTRXMPPTorAccount class]]) {
         cell.textView.dataDetectorTypes = UIDataDetectorTypeNone;
     }
@@ -1564,6 +1566,9 @@ - (NSAttributedString *)collectionView:(JSQMessagesCollectionView *)collectionVi
         [self.readOnlyDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
             mediaItem = [OTRMediaItem fetchObjectWithUniqueID:[message messageMediaItemKey] transaction:transaction];
         }];
+        if (!mediaItem) {
+            return attributedString;
+        }
 
         float percentProgress = mediaItem.transferProgress * 100;
 

OTRAssets/Strings/OTRStrings.h

@@ -30,6 +30,10 @@ FOUNDATION_EXPORT NSString* ARCHIVE_STRING();
 FOUNDATION_EXPORT NSString* ARE_YOU_SURE_STRING();
 /** "Audio Message", Text for media message summary */
 FOUNDATION_EXPORT NSString* AUDIO_MESSAGE_STRING();
+/** "Automatically Fetch Media", Title for other miscellaneous settings group */
+FOUNDATION_EXPORT NSString* AUTO_URL_FETCH_STRING();
+/** "All incoming messages containing URLs will be fetched by default to show a media preview. This is required for media messaging to work properly. Disable this if you do not trust your contacts.", Title for other miscellaneous settings group */
+FOUNDATION_EXPORT NSString* AUTO_URL_FETCH_WARNING_STRING();
 /** "Available", Label in buddy list for users that are available */
 FOUNDATION_EXPORT NSString* AVAILABLE_STRING();
 /** "Away", Label in buddy list for users that are away */

OTRAssets/Strings/OTRStrings.m

@@ -30,6 +30,10 @@
 NSString* ARE_YOU_SURE_STRING() { return [OTRLanguageManager translatedString:@"Are you sure?"]; }
 /** "Audio Message", Text for media message summary */
 NSString* AUDIO_MESSAGE_STRING() { return [OTRLanguageManager translatedString:@"Audio Message"]; }
+/** "Automatically Fetch Media", Title for other miscellaneous settings group */
+NSString* AUTO_URL_FETCH_STRING() { return [OTRLanguageManager translatedString:@"Automatically Fetch Media"]; }
+/** "All incoming messages containing URLs will be fetched by default to show a media preview. This is required for media messaging to work properly. Disable this if you do not trust your contacts.", Title for other miscellaneous settings group */
+NSString* AUTO_URL_FETCH_WARNING_STRING() { return [OTRLanguageManager translatedString:@"All incoming messages containing URLs will be fetched by default to show a media preview. This is required for media messaging to work properly. Disable this if you do not trust your contacts."]; }
 /** "Available", Label in buddy list for users that are available */
 NSString* AVAILABLE_STRING() { return [OTRLanguageManager translatedString:@"Available"]; }
 /** "Away", Label in buddy list for users that are away */