simplify wordcount example

lrgoodboy · Feb 12, 2013 · 1ffff9d · 1ffff9d
1 parent 2a6e929
commit 1ffff9d
Show file tree

Hide file tree

Showing 7 changed files with 44 additions and 67 deletions.
diff --git a/README.md b/README.md
@@ -2,19 +2,27 @@
 
 ## Usage
 
-### src/cia-hadoop/wordcount5.clj
+### Import into Eclipse
 
-* Compile
+    $ lein eclipse
+
+### Wordcount Example
+
+* Run locally with `lein`:
+
+    $ lein test
+
+* Compile:
 
     $ lein uberjar
 
-* Run locally:
+* Run locally with `java`:
 
-    $ java -cp target/cia-hadoop-0.1.0-SNAPSHOT-standalone.jar clojure_hadoop.job -job cia-hadoop.wordcount5/job -input README.md -output out5
+    $ java -cp target/cia-hadoop-0.1.0-SNAPSHOT-standalone.jar clojure_hadoop.job -job cia-hadoop.wordcount/job
 
 * Run on a cluster:
 
-    $ hadoop jar target/cia-hadoop-0.1.0-SNAPSHOT-standalone.jar clojure_hadoop.job -job cia-hadoop.wordcount5/job -input README.md -output out5
+    $ hadoop jar target/cia-hadoop-0.1.0-SNAPSHOT-standalone.jar clojure_hadoop.job -job cia-hadoop.wordcount/job
 
 ## License
 

diff --git a/project.clj b/project.clj
@@ -5,5 +5,4 @@
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure "1.4.0"]
                  [clojure-hadoop "1.4.1"]]
-  :plugins [[lein2-eclipse "2.0.0"]]
-  :aot [cia-hadoop.wordcount5])
+  :plugins [[lein2-eclipse "2.0.0"]])
diff --git a/src/cia_hadoop/core.clj b/src/cia_hadoop/core.clj
diff --git a/src/cia_hadoop/wordcount.clj b/src/cia_hadoop/wordcount.clj
@@ -0,0 +1,23 @@
+(ns cia-hadoop.wordcount
+  (:require [clojure-hadoop.wrap :as wrap]
+            [clojure-hadoop.defjob :as defjob])
+  (:import [java.util StringTokenizer])
+  (:use clojure-hadoop.job))
+
+(defn my-map [key value]
+  (map (fn [token] [token 1])
+       (enumeration-seq (StringTokenizer. value))))
+
+(defn my-reduce [key values-fn]
+  [[key (reduce + (values-fn))]])
+
+(defjob/defjob job
+  :map my-map
+  :map-reader wrap/int-string-map-reader
+  :reduce my-reduce
+  :input-format :text
+  :output-format :text
+  :compress-output false
+  :replace true
+  :input "README.md"
+  :output "out-wordcount")
diff --git a/src/cia_hadoop/wordcount5.clj b/src/cia_hadoop/wordcount5.clj
diff --git a/test/cia_hadoop/core_test.clj b/test/cia_hadoop/core_test.clj
diff --git a/test/cia_hadoop/wordcount_test.clj b/test/cia_hadoop/wordcount_test.clj
@@ -0,0 +1,7 @@
+(ns cia-hadoop.wordcount-test
+  (:use clojure.test
+        clojure-hadoop.job
+        cia-hadoop.wordcount))
+
+(deftest test-wordcount
+  (is (run job)))