Add MX record validation to import script.

ChronosWS · Feb 8, 2015 · 2a1ddab · 2a1ddab
1 parent fc10c67
commit 2a1ddab
Showing 1 changed file with 50 additions and 4 deletions.
diff --git a/update b/update
@@ -2,19 +2,65 @@
 
 set -e
 
-sources=$(cat ./data/sources.txt)
+tmp=$(mktemp -t freemail)
+cat ./data/blacklist.txt \
+    | sed '/./,$!d' \
+    | sed -e 's/^ *//' -e 's/ *$//' \
+    | awk '{print tolower($0)}' \
+    | sort \
+    | uniq > $tmp
+mv $tmp ./data/blacklist.txt
+
+tmp=$(mktemp -t freemail)
+cat ./data/free.txt \
+    | sed '/./,$!d' \
+    | sed -e 's/^ *//' -e 's/ *$//' \
+    | awk '{print tolower($0)}' \
+    | sort \
+    | uniq \
+    | comm -23 - ./data/blacklist.txt > $tmp
+mv $tmp ./data/free.txt
 
+tmp=$(mktemp -t freemail)
+cat ./data/disposable.txt \
+    | sed '/./,$!d' \
+    | sed -e 's/^ *//' -e 's/ *$//' \
+    | awk '{print tolower($0)}' \
+    | sort \
+    | uniq \
+    | comm -23 - ./data/blacklist.txt \
+    | comm -23 - ./data/free.txt > $tmp
+mv $tmp ./data/disposable.txt
+
+sources=$(cat ./data/sources.txt)
+new=$(mktemp -t freemail)
 for source in $sources; do
-    curl --silent $source >> tmp
+    echo "$(curl --silent $source)" >> $new
 done;
 
-cat data/free.txt tmp \
+tmp=$(mktemp -t freemail)
+cat $new \
     | sed '/./,$!d' \
     | sed -e 's/^ *//' -e 's/ *$//' \
     | awk '{print tolower($0)}' \
     | sort \
     | uniq \
     | comm -23 - ./data/blacklist.txt \
-    | comm -23 - ./data/disposable.txt > ./data/free.txt
+    | comm -23 - ./data/free.txt \
+    | comm -23 - ./data/disposable.txt > $tmp
+
+confirmed=$(mktemp -t freemail)
+for domain in $(cat $tmp); do
+    result=`dig +short mx $domain`
+    if [ -n "$result" ]; then
+        echo $domain >> $confirmed
+    fi
+done
+
+tmp=$(mktemp -t freemail)
+cat $confirmed ./data/free.txt \
+    | sort \
+    | uniq > $tmp
+mv $tmp ./data/free.txt
 
 echo 'Complete!'