update to use TDS (duckduckgo#533)

Author: brindy

Core/AppUrls.swift

@@ -28,24 +28,24 @@ public struct AppUrls {
         }
 
         static let base = ProcessInfo.processInfo.environment["BASE_URL", default: "https://duckduckgo.com"]
+        static let staticBase = "https://staticcdn.duckduckgo.com"
+        
         static let autocomplete = "\(base)/ac/"
-        static let disconnectMeBlockList = "\(base)/contentblocking.js?l=disconnect"
-        static let easylistBlockList = "\(base)/contentblocking.js?l=easylist"
-        static let easylistPrivacyBlockList = "\(base)/contentblocking.js?l=easyprivacy"
-        static let trackersWhitelist = "\(base)/contentblocking/trackers-whitelist.txt"
+        
         static let surrogates = "\(base)/contentblocking.js?l=surrogates"
-        static let entitylist = "\(base)/contentblocking.js?l=entitylist2"
+        static let temporaryWhitelist = "\(base)/contentblocking/trackers-whitelist-temporary.txt"
+        static let trackerDataSet = "\(staticBase)/trackerblocking/tds.json"
+
         static let atb = "\(base)/atb.js\(devMode)"
         static let exti = "\(base)/exti/\(devMode)"
         static let feedback = "\(base)/feedback.js?type=app-feedback"
         static let faviconService = "\(base)/ip3/%@.ico"
 
-        static let staticBase = "https://staticcdn.duckduckgo.com"
         static let httpsBloomFilter = "\(staticBase)/https/https-mobile-bloom.bin?cache-version=1"
         static let httpsBloomFilterSpec = "\(staticBase)/https/https-mobile-bloom-spec.json?cache-version=1"
         static let httpsWhitelist = "\(staticBase)/https/https-mobile-whitelist.json?cache-version=1"
         static let httpsLookupService = "\(base)/smarter_encryption.js"
-        
+
         static let pixelBase = ProcessInfo.processInfo.environment["PIXEL_BASE_URL", default: "https://improving.duckduckgo.com"]
         static let pixel = "\(pixelBase)/t/%@_ios_%@"
     }
@@ -78,28 +78,16 @@ public struct AppUrls {
         return URL(string: Url.autocomplete)!.addParam(name: Param.search, value: text)
     }
 
-    public var disconnectMeBlockList: URL {
-        return URL(string: Url.disconnectMeBlockList)!
-    }
-
-    public var easylistBlockList: URL {
-        return URL(string: Url.easylistBlockList)!
-    }
-
-    public var easylistPrivacyBlockList: URL {
-        return URL(string: Url.easylistPrivacyBlockList)!
-    }
-    
-    public var trackersWhitelist: URL {
-        return URL(string: Url.trackersWhitelist)!
-    }
-
     public var surrogates: URL {
         return URL(string: Url.surrogates)!
     }
-
-    public var entitylist: URL {
-        return URL(string: Url.entitylist)!
+    
+    public var trackerDataSet: URL {
+        return URL(string: Url.trackerDataSet)!
+    }
+    
+    public var temporaryWhitelist: URL {
+        return URL(string: Url.temporaryWhitelist)!
     }
 
     public var feedback: URL {

Core/CohortRequest.swift

@@ -26,24 +26,23 @@ public class ContentBlockerLoader {
 
     private let httpsUpgradeStore: HTTPSUpgradeStore = HTTPSUpgradePersistence()
     private let etagStorage: BlockerListETagStorage
+    private let fileStore: FileStore
 
     private var newData = DataDict()
     private var etags = EtagDict()
 
-    init(etagStorage: BlockerListETagStorage = UserDefaultsETagStorage()) {
+    init(etagStorage: BlockerListETagStorage = UserDefaultsETagStorage(), fileStore: FileStore = FileStore()) {
         self.etagStorage = etagStorage
+        self.fileStore = fileStore
     }
 
-    func checkForUpdates(with store: EtagOOSCheckStore,
-                         dataSource: ContentBlockerRemoteDataSource = ContentBlockerRequest()) -> Bool {
+    func checkForUpdates(dataSource: ContentBlockerRemoteDataSource = ContentBlockerRequest()) -> Bool {
 
-        EasylistStore.removeLegacyLists()
-
         self.newData.removeAll()
         self.etags.removeAll()
 
         let semaphore = DispatchSemaphore(value: 0)
-        let numberOfRequests = startRequests(with: semaphore, store: store, dataSource: dataSource)
+        let numberOfRequests = startRequests(with: semaphore, dataSource: dataSource)
 
         for _ in 0 ..< numberOfRequests {
             semaphore.wait()
@@ -57,8 +56,7 @@ public class ContentBlockerLoader {
     func applyUpdate(to cache: StorageCacheUpdating) {
 
         for (config, info) in newData {
-            if (cache.update(config, with: info)),
-                let etag = etags[config] {
+            if cache.update(config, with: info), let etag = etags[config] {
                 etagStorage.set(etag: etag, for: config)
             } else {
                 Logger.log(text: "Failed to apply update to \(config.rawValue)")
@@ -67,13 +65,11 @@ public class ContentBlockerLoader {
     }
 
     private func startRequests(with semaphore: DispatchSemaphore,
-                               store: EtagOOSCheckStore,
                                dataSource: ContentBlockerRemoteDataSource) -> Int {
-        request(.entitylist, with: dataSource, store: store, semaphore)
-        request(.disconnectMe, with: dataSource, store: store, semaphore)
-        request(.trackersWhitelist, with: dataSource, store: store, semaphore)
-        request(.surrogates, with: dataSource, store: store, semaphore)
 
+        request(.surrogates, with: dataSource, semaphore)
+        request(.trackerDataSet, with: dataSource, semaphore)
+        request(.temporaryWhitelist, with: dataSource, semaphore)
         requestHttpsUpgrade(dataSource, semaphore)
         requestHttpsWhitelist(dataSource, semaphore)
 
@@ -82,7 +78,6 @@ public class ContentBlockerLoader {
 
     fileprivate func request(_ configuration: ContentBlockerRequest.Configuration,
                              with contentBlockerRequest: ContentBlockerRemoteDataSource,
-                             store: EtagOOSCheckStore,
                              _ semaphore: DispatchSemaphore) {
         contentBlockerRequest.request(configuration) { response in
 
@@ -94,22 +89,7 @@ public class ContentBlockerLoader {
             let isCached = etag != nil && self.etagStorage.etag(for: configuration) == etag
             self.etags[configuration] = etag
 
-            if isCached {
-                switch configuration {
-                case .disconnectMe:
-                    if !store.hasDisconnectMeData {
-                        self.newData[configuration] = data
-                        Pixel.fire(pixel: .etagStoreOOSWithDisconnectMeFix)
-                    }
-                case .trackersWhitelist:
-                    if !store.hasEasylistData {
-                        self.newData[configuration] = data
-                        Pixel.fire(pixel: .etagStoreOOSWithEasylistFix)
-                    }
-                default:
-                    break
-                }
-            } else {
+            if !isCached || !self.fileStore.hasData(forConfiguration: configuration) {
                 self.newData[configuration] = data
             }
 

Core/ContentBlockerLoader.swift

@@ -32,17 +32,14 @@ class ContentBlockerRequest: ContentBlockerRemoteDataSource {
         case error
         case success(etag: String?, data: Data)
     }
-    
+
     enum Configuration: String {
-        case disconnectMe = "disconnectme"
-        case easylist = "easylist"
-        case easylistPrivacy = "easyprivacy"
-        case trackersWhitelist
         case httpsBloomFilterSpec
         case httpsBloomFilter
         case httpsWhitelist
         case surrogates
-        case entitylist = "entitylist2"
+        case trackerDataSet
+        case temporaryWhitelist
     }
 
     var requestCount = 0
@@ -71,15 +68,12 @@ class ContentBlockerRequest: ContentBlockerRemoteDataSource {
         let appUrls = AppUrls()
 
         switch list {
-        case .disconnectMe: return appUrls.disconnectMeBlockList
-        case .easylist: return appUrls.easylistBlockList
-        case .easylistPrivacy: return appUrls.easylistPrivacyBlockList
         case .httpsBloomFilterSpec: return appUrls.httpsBloomFilterSpec
         case .httpsBloomFilter: return appUrls.httpsBloomFilter
         case .httpsWhitelist: return appUrls.httpsWhitelist
-        case .trackersWhitelist: return appUrls.trackersWhitelist
         case .surrogates: return appUrls.surrogates
-        case .entitylist: return appUrls.entitylist
+        case .trackerDataSet: return appUrls.trackerDataSet
+        case .temporaryWhitelist: return appUrls.temporaryWhitelist
         }
     }
 }

Core/ContentBlockerRequest.swift

@@ -21,50 +21,11 @@ import Foundation
 
 public class ContentBlockerStringCache {
 
-    struct Constants {
-        // bump the cache version if you know the cache should be invalidated on the next release
-        static let cacheVersion = 5
-        static let cacheVersionKey = "com.duckduckgo.contentblockerstringcache.version"
-    }
-
-    private var cacheDir: URL {
+    public static func removeLegacyData() {
+        let fileManager = FileManager.default
         let groupName = ContentBlockerStoreConstants.groupName
-        return fileManager.containerURL(forSecurityApplicationGroupIdentifier: groupName)!.appendingPathComponent("string-cache")
-    }
-
-    private var fileManager: FileManager {
-        return FileManager.default
-    }
-
-    public init(userDefaults: UserDefaults = UserDefaults.standard) {
-        let lastSeenVersion = userDefaults.integer(forKey: Constants.cacheVersionKey)
-        if lastSeenVersion < Constants.cacheVersion {
-            clearCache()
-            userDefaults.set(Constants.cacheVersion, forKey: Constants.cacheVersionKey)
-        }
-    }
-
-    private func clearCache() {
+        let cacheDir = fileManager.containerURL(forSecurityApplicationGroupIdentifier: groupName)!.appendingPathComponent("string-cache")
         try? fileManager.removeItem(atPath: cacheDir.path)
     }
 
-    public func get(named name: String) -> String? {
-        return try? String(contentsOf: persistenceLocation(for: name), encoding: .utf8)
-    }
-
-    public func put(name: String, value: String) {
-        try? value.write(to: persistenceLocation(for: name), atomically: true, encoding: .utf8)
-    }
-
-    public func remove(named name: String) {
-        try? fileManager.removeItem(at: persistenceLocation(for: name))
-    }
-
-    private func persistenceLocation(for name: String) -> URL {
-        try? fileManager.createDirectory(at: cacheDir, withIntermediateDirectories: true, attributes: nil)
-        let location = cacheDir.appendingPathComponent(name)
-        Logger.log(text: "cache \(name) \(location)")
-        return location
-    }
-
 }

Core/ContentBlockerStringCache.swift

@@ -19,49 +19,41 @@
 
 import Foundation
 
-// Populated with relevant info at the point of detection.  If networkName or category are nil, they are genuinely not known.
+// Populated with relevant info at the point of detection.
 public struct DetectedTracker {
 
     public let url: String
+    public let knownTracker: KnownTracker?
+    public let entity: Entity?
     public let blocked: Bool
-    public let networkName: String?
-    public let category: String?
-
-    public init(url: String, networkName: String?, category: String?, blocked: Bool) {
+    
+    public init(url: String, knownTracker: KnownTracker?, entity: Entity?, blocked: Bool) {
         self.url = url
-        self.networkName = networkName
-        self.category = category
+        self.knownTracker = knownTracker
+        self.entity = entity
         self.blocked = blocked
     }
 
     public var domain: String? {
         return URL(string: url)?.host
     }
-
-    public var isIpTracker: Bool {
-        return URL.isValidIpHost(domain ?? "")
-    }
 
     public var networkNameForDisplay: String {
-        return networkName ?? domain ?? url
+        return entity?.displayName ?? domain ?? url
     }
 
 }
 
-extension DetectedTracker: Hashable {
-
-    public func hash( into hasher: inout Hasher) {
-        hasher.combine(url)
-        hasher.combine(blocked)
-        hasher.combine(networkName)
-        hasher.combine(category)
-    }
-
+extension DetectedTracker: Hashable, Equatable {
+    
     public static func == (lhs: DetectedTracker, rhs: DetectedTracker) -> Bool {
-        return lhs.url == rhs.url
-            && lhs.blocked == rhs.blocked
-            && lhs.networkName == rhs.networkName
-            && lhs.category == rhs.category
+        return ((lhs.entity != nil || rhs.entity != nil) && lhs.entity?.displayName == rhs.entity?.displayName)
+            && lhs.domain ?? "" == rhs.domain ?? ""
     }
-
+    
+    public func hash(into hasher: inout Hasher) {
+        hasher.combine(self.entity?.displayName)
+        hasher.combine(self.domain)
+    }
+    
 }