spark port intial commit

spark/build.sbt

@@ -0,0 +1,25 @@
+import sbtassembly.{Plugin},Plugin.AssemblyKeys._
+import sbtassembly.Plugin._
+
+name := "beymani"
+
+organization := "org.beymani"
+
+version := "1.0"
+
+scalaVersion := "2.10.4"
+
+libraryDependencies ++=Seq(
+  "org.apache.spark" %% "spark-core" % "1.1.0" % "provided",
+  "org.apache.spark" %% "spark-streaming" % "1.1.0",
+  "org.apache.spark" %% "spark-streaming-kafka" % "1.1.0",
+  "org.apache.commons" % "commons-lang3" % "3.0",
+  "com.fasterxml.jackson.core" % "jackson-databind" % "2.3.3",
+  "com.fasterxml.jackson.module" % "jackson-module-scala_2.10" % "2.3.3",
+  "junit" % "junit" % "4.7" % "test",
+  "org.scalatest" % "scalatest_2.10" % "2.0" % "test"
+)
+
+net.virtualvoid.sbt.graph.Plugin.graphSettings
+
+assemblySettings

spark/project/build.properties

@@ -0,0 +1 @@
+sbt.version=0.13.0

spark/project/plugins.sbt

@@ -0,0 +1,15 @@
+logLevel := Level.Warn
+
+addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.7.4")
+
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
+
+addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "2.5.0")
+
+resolvers ++= Seq(
+  "Cloudera Repository" at "https://repository.cloudera.com/artifactory/cloudera-repos/",
+  "Akka Repository" at "http://repo.akka.io/releases/",
+  "scala-tools" at "https://oss.sonatype.org/content/groups/scala-tools"
+)
+
+

spark/src/main/scala/org/beymani/sanity/WordCount.scala

@@ -0,0 +1,29 @@
+package org.beymani.sanity
+
+import org.apache.spark._
+import org.apache.spark.SparkContext._
+
+object WordCount {
+    def main(args: Array[String]) {
+      val master = args.length match {
+        case x: Int if x > 0 => args(0)
+        case _ => "local"
+      }
+      val sc = new SparkContext(master, "WordCount", System.getenv("SPARK_HOME"))
+      val input = args.length match {
+        case x: Int if x > 1 => sc.textFile(args(1))
+        case _ => sc.parallelize(List("pandas", "i like pandas"))
+      }
+      val words = input.flatMap(line => line.split(" "))
+      args.length match {
+        case x: Int if x > 2 => {
+          val counts = words.map(word => (word, 1)).reduceByKey{case (x,y) => x + y}
+          counts.saveAsTextFile(args(2))
+        }
+        case _ => {
+          val wc = words.countByValue()
+          println(wc.mkString(","))
+        }
+      }
+    }
+}

spark/version.sbt

@@ -0,0 +1 @@
+version in ThisBuild := "1.0-SNAPSHOT"