spark-submit updates

Author: bllchmbrs

project-templates/java/src/main/java/com/databricks/example/SimpleExample.java

@@ -1,4 +1,4 @@
-package main.example;
+package com.databricks.example;
 
 import org.apache.spark.sql.SparkSession;
 
@@ -12,7 +12,7 @@ public static void main(String[] args) {
                 .builder()
                 .getOrCreate();
 
-        spark.range(1, 2000).count();
+        System.out.println(spark.range(1, 2000).count());
 
     }
 }

project-templates/python/pyspark_template/main.py

@@ -1,6 +1,4 @@
-
-
-
+from __future__ import print_function
 
 if __name__ == '__main__':
 
@@ -11,3 +9,5 @@
         .appName("Word Count") \
         .config("spark.some.config.option", "some-value") \
         .getOrCreate()
+        
+    print(spark.range(5000).where("id > 500").selectExpr("sum(id)").collect())

project-templates/scala/.gitignore

@@ -1,2 +1,2 @@
-project/target/*
+project/*
 target/*

project-templates/scala/build.sbt

@@ -45,29 +45,30 @@ libraryDependencies ++= Seq(
 //////////
 
 // Your username to login to Databricks
-dbcUsername := sys.env("DATABRICKSUSERNAME")
+val dbcUsername = sys.env("DATABRICKSUSERNAME")
 
 // Your password (Can be set as an environment variable)
-dbcPassword := sys.env("DATABRICKSPASSWORD")
+val dbcPassword = sys.env("DATABRICKSPASSWORD")
 // Gotcha: Setting environment variables in IDE's may differ.
 // IDE's usually don't pick up environment variables from .bash_profile or .bashrc
 
 // The URL to the Databricks REST API
-dbcApiUrl := "https://your-sub-domain.cloud.databricks.com/api/1.2"
+val dbcApiUrl = "https://your-sub-domain.cloud.databricks.com/api/1.2"
 
 // Add any clusters that you would like to deploy your work to. e.g. "My Cluster"
-dbcClusters += "my-cluster" // Add "ALL_CLUSTERS" if you want to attach your work to all clusters
+val dbcClusters = Seq("my-cluster")
+// Add "ALL_CLUSTERS" if you want to attach your work to all clusters
 
 // An optional parameter to set the location to upload your libraries to in the workspace
 // e.g. "/Shared/libraries"
 // This location must be an existing path and all folders must exist.
 // NOTE: Specifying this parameter is *strongly* recommended as many jars will be uploaded to your cluster.
 // Putting them in one folder will make it easy for your to delete all the libraries at once.
 // Default is "/"
-dbcLibraryPath := "/Shared/Libraries"
+val dbcLibraryPath = "/Shared/Libraries"
 
 // Whether to restart the clusters everytime a new version is uploaded to Databricks.
-dbcRestartOnAttach := false // Default true
+val dbcRestartOnAttach = false // Default true
 
 //////////
 ///// END Databricks Settings