diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..f57b2e9 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..286ca11 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..60c6d36 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..12b7786 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + Android + + + CorrectnessLintAndroid + + + General + + + Gradle + + + Kotlin + + + LintAndroid + + + Maven + + + OSGi + + + Probable bugsGradle + + + + + Duplicates + + + + + + + + + \ No newline at end of file diff --git a/.idea/scala_compiler.xml b/.idea/scala_compiler.xml new file mode 100644 index 0000000..211d2e4 --- /dev/null +++ b/.idea/scala_compiler.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/spark.iml b/.idea/spark.iml new file mode 100644 index 0000000..78b2cc5 --- /dev/null +++ b/.idea/spark.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..e96534f --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ck/.checkpoint-1545878800000.bk.crc b/ck/.checkpoint-1545878800000.bk.crc new file mode 100644 index 0000000..14a5c15 Binary files /dev/null and b/ck/.checkpoint-1545878800000.bk.crc differ diff --git a/ck/.checkpoint-1545878800000.crc b/ck/.checkpoint-1545878800000.crc new file mode 100644 index 0000000..4d7d47b Binary files /dev/null and b/ck/.checkpoint-1545878800000.crc differ diff --git a/ck/.checkpoint-1545878805000.bk.crc b/ck/.checkpoint-1545878805000.bk.crc new file mode 100644 index 0000000..132ce02 Binary files /dev/null and b/ck/.checkpoint-1545878805000.bk.crc differ diff --git a/ck/.checkpoint-1545878805000.crc b/ck/.checkpoint-1545878805000.crc new file mode 100644 index 0000000..c5de69c Binary files /dev/null and b/ck/.checkpoint-1545878805000.crc differ diff --git a/ck/.checkpoint-1545878810000.bk.crc b/ck/.checkpoint-1545878810000.bk.crc new file mode 100644 index 0000000..2024d52 Binary files /dev/null and b/ck/.checkpoint-1545878810000.bk.crc differ diff --git a/ck/.checkpoint-1545878810000.crc b/ck/.checkpoint-1545878810000.crc new file mode 100644 index 0000000..22130b3 Binary files /dev/null and b/ck/.checkpoint-1545878810000.crc differ diff --git a/ck/.checkpoint-1545878815000.bk.crc b/ck/.checkpoint-1545878815000.bk.crc new file mode 100644 index 0000000..f963b23 Binary files /dev/null and b/ck/.checkpoint-1545878815000.bk.crc differ diff --git a/ck/.checkpoint-1545878815000.crc b/ck/.checkpoint-1545878815000.crc new file mode 100644 index 0000000..c22bb97 Binary files /dev/null and b/ck/.checkpoint-1545878815000.crc differ diff --git a/ck/.checkpoint-1545878820000.bk.crc b/ck/.checkpoint-1545878820000.bk.crc new file mode 100644 index 0000000..adb88c3 Binary files /dev/null and b/ck/.checkpoint-1545878820000.bk.crc differ diff --git a/ck/.checkpoint-1545878820000.crc b/ck/.checkpoint-1545878820000.crc new file mode 100644 index 0000000..f7a4ec2 Binary files /dev/null and b/ck/.checkpoint-1545878820000.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/._partitioner.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/._partitioner.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00000.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00000.crc new file mode 100644 index 0000000..2bbc778 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00000.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00001.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00001.crc new file mode 100644 index 0000000..6a5b536 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00001.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00002.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00002.crc new file mode 100644 index 0000000..fd0849e Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00002.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00003.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00003.crc new file mode 100644 index 0000000..c935799 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/.part-00003.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/_partitioner b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/_partitioner differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00000 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00000 new file mode 100644 index 0000000..b3cba5f Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00000 differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00001 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00001 new file mode 100644 index 0000000..34cf844 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00001 differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00002 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00002 new file mode 100644 index 0000000..62cf148 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00002 differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00003 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00003 new file mode 100644 index 0000000..498d985 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-43/part-00003 differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/._partitioner.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/._partitioner.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00000.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00000.crc new file mode 100644 index 0000000..2bbc778 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00000.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00001.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00001.crc new file mode 100644 index 0000000..6a5b536 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00001.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00002.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00002.crc new file mode 100644 index 0000000..fd0849e Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00002.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00003.crc b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00003.crc new file mode 100644 index 0000000..c935799 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/.part-00003.crc differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/_partitioner b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/_partitioner differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00000 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00000 new file mode 100644 index 0000000..b3cba5f Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00000 differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00001 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00001 new file mode 100644 index 0000000..34cf844 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00001 differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00002 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00002 new file mode 100644 index 0000000..62cf148 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00002 differ diff --git a/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00003 b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00003 new file mode 100644 index 0000000..498d985 Binary files /dev/null and b/ck/3daf2bff-77e9-4dde-8e53-f64bc2ca0366/rdd-58/part-00003 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/._partitioner.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/._partitioner.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00000.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00000.crc new file mode 100644 index 0000000..cb9747a Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00000.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00001.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00001.crc new file mode 100644 index 0000000..a287e4d Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00001.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00002.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00002.crc new file mode 100644 index 0000000..15cab98 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00002.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00003.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00003.crc new file mode 100644 index 0000000..cad9aa3 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/.part-00003.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/_partitioner b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/_partitioner differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00000 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00000 new file mode 100644 index 0000000..edbd4c0 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00000 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00001 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00001 new file mode 100644 index 0000000..a3df462 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00001 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00002 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00002 new file mode 100644 index 0000000..711006c Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00002 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00003 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00003 new file mode 100644 index 0000000..e4e4717 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-73/part-00003 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/._partitioner.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/._partitioner.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00000.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00000.crc new file mode 100644 index 0000000..cb9747a Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00000.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00001.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00001.crc new file mode 100644 index 0000000..a287e4d Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00001.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00002.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00002.crc new file mode 100644 index 0000000..15cab98 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00002.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00003.crc b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00003.crc new file mode 100644 index 0000000..cad9aa3 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/.part-00003.crc differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/_partitioner b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/_partitioner differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00000 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00000 new file mode 100644 index 0000000..edbd4c0 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00000 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00001 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00001 new file mode 100644 index 0000000..a3df462 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00001 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00002 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00002 new file mode 100644 index 0000000..711006c Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00002 differ diff --git a/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00003 b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00003 new file mode 100644 index 0000000..e4e4717 Binary files /dev/null and b/ck/6a656713-43cd-41c9-8a44-f9baca5e5068/rdd-88/part-00003 differ diff --git a/ck/checkpoint-1545878800000 b/ck/checkpoint-1545878800000 new file mode 100644 index 0000000..3688263 Binary files /dev/null and b/ck/checkpoint-1545878800000 differ diff --git a/ck/checkpoint-1545878800000.bk b/ck/checkpoint-1545878800000.bk new file mode 100644 index 0000000..875a04f Binary files /dev/null and b/ck/checkpoint-1545878800000.bk differ diff --git a/ck/checkpoint-1545878805000 b/ck/checkpoint-1545878805000 new file mode 100644 index 0000000..36ccd8f Binary files /dev/null and b/ck/checkpoint-1545878805000 differ diff --git a/ck/checkpoint-1545878805000.bk b/ck/checkpoint-1545878805000.bk new file mode 100644 index 0000000..3a97c87 Binary files /dev/null and b/ck/checkpoint-1545878805000.bk differ diff --git a/ck/checkpoint-1545878810000 b/ck/checkpoint-1545878810000 new file mode 100644 index 0000000..d6f9936 Binary files /dev/null and b/ck/checkpoint-1545878810000 differ diff --git a/ck/checkpoint-1545878810000.bk b/ck/checkpoint-1545878810000.bk new file mode 100644 index 0000000..6572c00 Binary files /dev/null and b/ck/checkpoint-1545878810000.bk differ diff --git a/ck/checkpoint-1545878815000 b/ck/checkpoint-1545878815000 new file mode 100644 index 0000000..7261822 Binary files /dev/null and b/ck/checkpoint-1545878815000 differ diff --git a/ck/checkpoint-1545878815000.bk b/ck/checkpoint-1545878815000.bk new file mode 100644 index 0000000..0700df4 Binary files /dev/null and b/ck/checkpoint-1545878815000.bk differ diff --git a/ck/checkpoint-1545878820000 b/ck/checkpoint-1545878820000 new file mode 100644 index 0000000..6a92fb1 Binary files /dev/null and b/ck/checkpoint-1545878820000 differ diff --git a/ck/checkpoint-1545878820000.bk b/ck/checkpoint-1545878820000.bk new file mode 100644 index 0000000..5f6b2c0 Binary files /dev/null and b/ck/checkpoint-1545878820000.bk differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/._partitioner.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/._partitioner.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00000.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00000.crc new file mode 100644 index 0000000..b47df20 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00000.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00001.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00001.crc new file mode 100644 index 0000000..ca1f2e4 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00001.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00002.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00002.crc new file mode 100644 index 0000000..35f231f Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00002.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00003.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00003.crc new file mode 100644 index 0000000..b94d8a8 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/.part-00003.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/_partitioner b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/_partitioner differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00000 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00000 new file mode 100644 index 0000000..af469c6 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00000 differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00001 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00001 new file mode 100644 index 0000000..b669cb0 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00001 differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00002 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00002 new file mode 100644 index 0000000..dd4cee5 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00002 differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00003 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00003 new file mode 100644 index 0000000..5ff1c32 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-43/part-00003 differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/._partitioner.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/._partitioner.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00000.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00000.crc new file mode 100644 index 0000000..b47df20 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00000.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00001.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00001.crc new file mode 100644 index 0000000..ca1f2e4 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00001.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00002.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00002.crc new file mode 100644 index 0000000..35f231f Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00002.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00003.crc b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00003.crc new file mode 100644 index 0000000..b94d8a8 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/.part-00003.crc differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/_partitioner b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/_partitioner differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00000 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00000 new file mode 100644 index 0000000..af469c6 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00000 differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00001 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00001 new file mode 100644 index 0000000..b669cb0 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00001 differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00002 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00002 new file mode 100644 index 0000000..dd4cee5 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00002 differ diff --git a/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00003 b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00003 new file mode 100644 index 0000000..5ff1c32 Binary files /dev/null and b/ck/db456e73-94a9-4192-9d55-96f5ef878ea1/rdd-58/part-00003 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/._partitioner.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/._partitioner.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00000.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00000.crc new file mode 100644 index 0000000..15cab98 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00000.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00001.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00001.crc new file mode 100644 index 0000000..b3e5ed7 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00001.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00002.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00002.crc new file mode 100644 index 0000000..15cab98 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00002.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00003.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00003.crc new file mode 100644 index 0000000..15cab98 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/.part-00003.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/_partitioner b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/_partitioner differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00000 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00000 new file mode 100644 index 0000000..711006c Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00000 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00001 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00001 new file mode 100644 index 0000000..d4c77d9 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00001 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00002 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00002 new file mode 100644 index 0000000..711006c Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00002 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00003 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00003 new file mode 100644 index 0000000..711006c Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-28/part-00003 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/._partitioner.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/._partitioner.crc new file mode 100644 index 0000000..8108233 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/._partitioner.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00000.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00000.crc new file mode 100644 index 0000000..15cab98 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00000.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00001.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00001.crc new file mode 100644 index 0000000..032b18d Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00001.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00002.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00002.crc new file mode 100644 index 0000000..8baf6c3 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00002.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00003.crc b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00003.crc new file mode 100644 index 0000000..15cab98 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/.part-00003.crc differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/_partitioner b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/_partitioner new file mode 100644 index 0000000..d32753c Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/_partitioner differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00000 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00000 new file mode 100644 index 0000000..711006c Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00000 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00001 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00001 new file mode 100644 index 0000000..d038d05 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00001 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00002 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00002 new file mode 100644 index 0000000..de5a5d8 Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00002 differ diff --git a/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00003 b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00003 new file mode 100644 index 0000000..711006c Binary files /dev/null and b/ck/e4491574-2259-4595-b1c6-55be6a088868/rdd-43/part-00003 differ diff --git a/ck/receivedBlockMetadata/log-1545878740413-1545878800413 b/ck/receivedBlockMetadata/log-1545878740413-1545878800413 new file mode 100644 index 0000000..a1e42f8 Binary files /dev/null and b/ck/receivedBlockMetadata/log-1545878740413-1545878800413 differ diff --git a/ck/receivedBlockMetadata/log-1545878795005-1545878855005 b/ck/receivedBlockMetadata/log-1545878795005-1545878855005 new file mode 100644 index 0000000..22cddb4 Binary files /dev/null and b/ck/receivedBlockMetadata/log-1545878795005-1545878855005 differ diff --git a/data/student.txt b/data/student.txt new file mode 100644 index 0000000..c661ff4 --- /dev/null +++ b/data/student.txt @@ -0,0 +1,6 @@ +1 zhangsan 20 +2 lisi 29 +3 wangwu 25 +4 zhaoliu 30 +5 tianqi 35 +6 kobe 40 diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml new file mode 100644 index 0000000..d68bda7 --- /dev/null +++ b/dependency-reduced-pom.xml @@ -0,0 +1,67 @@ + + + 4.0.0 + com.itcast + spark + 1.0-SNAPSHOT + + src/main/scala + src/test/scala + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + + compile + testCompile + + + + -dependencyfile + ${project.build.directory}/.scala_dependencies + + + + + + + maven-shade-plugin + 2.3 + + + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + + 2.11.8 + 2.7.4 + 2.0.2 + + + diff --git a/derby.log b/derby.log new file mode 100644 index 0000000..d564aa7 --- /dev/null +++ b/derby.log @@ -0,0 +1,13 @@ +---------------------------------------------------------------- +Tue Dec 25 10:38:29 CST 2018: +Booting Derby version The Apache Software Foundation - Apache Derby - 10.10.2.0 - (1582446): instance a816c00e-0167-e33a-504a-000008939228 +on database directory C:\Users\Administrator\IdeaProjects\spark\metastore_db with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@1a3e5f23 +Loaded from file:/C:/Program%20Files/repository_maven/org/apache/derby/derby/10.10.2.0/derby-10.10.2.0.jar +java.vendor=Oracle Corporation +java.runtime.version=1.8.0_181-b13 +user.dir=C:\Users\Administrator\IdeaProjects\spark +os.name=Windows 10 +os.arch=amd64 +os.version=10.0 +derby.system.home=null +Database Class Loader started - derby.database.classpath='' diff --git a/metastore_db/README_DO_NOT_TOUCH_FILES.txt b/metastore_db/README_DO_NOT_TOUCH_FILES.txt new file mode 100644 index 0000000..a4bc145 --- /dev/null +++ b/metastore_db/README_DO_NOT_TOUCH_FILES.txt @@ -0,0 +1,9 @@ + +# ************************************************************************* +# *** DO NOT TOUCH FILES IN THIS DIRECTORY! *** +# *** FILES IN THIS DIRECTORY AND SUBDIRECTORIES CONSTITUTE A DERBY *** +# *** DATABASE, WHICH INCLUDES THE DATA (USER AND SYSTEM) AND THE *** +# *** FILES NECESSARY FOR DATABASE RECOVERY. *** +# *** EDITING, ADDING, OR DELETING ANY OF THESE FILES MAY CAUSE DATA *** +# *** CORRUPTION AND LEAVE THE DATABASE IN A NON-RECOVERABLE STATE. *** +# ************************************************************************* \ No newline at end of file diff --git a/metastore_db/db.lck b/metastore_db/db.lck new file mode 100644 index 0000000..a33e39d Binary files /dev/null and b/metastore_db/db.lck differ diff --git a/metastore_db/log/README_DO_NOT_TOUCH_FILES.txt b/metastore_db/log/README_DO_NOT_TOUCH_FILES.txt new file mode 100644 index 0000000..56df292 --- /dev/null +++ b/metastore_db/log/README_DO_NOT_TOUCH_FILES.txt @@ -0,0 +1,8 @@ + +# ************************************************************************* +# *** DO NOT TOUCH FILES IN THIS DIRECTORY! *** +# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE RECOVERY *** +# *** SYSTEM. EDITING, ADDING, OR DELETING FILES IN THIS DIRECTORY *** +# *** WILL CAUSE THE DERBY RECOVERY SYSTEM TO FAIL, LEADING TO *** +# *** NON-RECOVERABLE CORRUPT DATABASES. *** +# ************************************************************************* \ No newline at end of file diff --git a/metastore_db/log/log.ctrl b/metastore_db/log/log.ctrl new file mode 100644 index 0000000..7297085 Binary files /dev/null and b/metastore_db/log/log.ctrl differ diff --git a/metastore_db/log/log1.dat b/metastore_db/log/log1.dat new file mode 100644 index 0000000..6d38ced Binary files /dev/null and b/metastore_db/log/log1.dat differ diff --git a/metastore_db/log/logmirror.ctrl b/metastore_db/log/logmirror.ctrl new file mode 100644 index 0000000..7297085 Binary files /dev/null and b/metastore_db/log/logmirror.ctrl differ diff --git a/metastore_db/seg0/README_DO_NOT_TOUCH_FILES.txt b/metastore_db/seg0/README_DO_NOT_TOUCH_FILES.txt new file mode 100644 index 0000000..2bdad06 --- /dev/null +++ b/metastore_db/seg0/README_DO_NOT_TOUCH_FILES.txt @@ -0,0 +1,8 @@ + +# ************************************************************************* +# *** DO NOT TOUCH FILES IN THIS DIRECTORY! *** +# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE TO STORE *** +# *** USER AND SYSTEM DATA. EDITING, ADDING, OR DELETING FILES IN THIS *** +# *** DIRECTORY WILL CORRUPT THE ASSOCIATED DERBY DATABASE AND MAKE *** +# *** IT NON-RECOVERABLE. *** +# ************************************************************************* \ No newline at end of file diff --git a/metastore_db/seg0/c10.dat b/metastore_db/seg0/c10.dat new file mode 100644 index 0000000..8d8e9ab Binary files /dev/null and b/metastore_db/seg0/c10.dat differ diff --git a/metastore_db/seg0/c101.dat b/metastore_db/seg0/c101.dat new file mode 100644 index 0000000..1118178 Binary files /dev/null and b/metastore_db/seg0/c101.dat differ diff --git a/metastore_db/seg0/c111.dat b/metastore_db/seg0/c111.dat new file mode 100644 index 0000000..bab1dfc Binary files /dev/null and b/metastore_db/seg0/c111.dat differ diff --git a/metastore_db/seg0/c121.dat b/metastore_db/seg0/c121.dat new file mode 100644 index 0000000..fbf18b9 Binary files /dev/null and b/metastore_db/seg0/c121.dat differ diff --git a/metastore_db/seg0/c130.dat b/metastore_db/seg0/c130.dat new file mode 100644 index 0000000..7b2a654 Binary files /dev/null and b/metastore_db/seg0/c130.dat differ diff --git a/metastore_db/seg0/c141.dat b/metastore_db/seg0/c141.dat new file mode 100644 index 0000000..3d983c2 Binary files /dev/null and b/metastore_db/seg0/c141.dat differ diff --git a/metastore_db/seg0/c150.dat b/metastore_db/seg0/c150.dat new file mode 100644 index 0000000..e2ea576 Binary files /dev/null and b/metastore_db/seg0/c150.dat differ diff --git a/metastore_db/seg0/c161.dat b/metastore_db/seg0/c161.dat new file mode 100644 index 0000000..f68dad2 Binary files /dev/null and b/metastore_db/seg0/c161.dat differ diff --git a/metastore_db/seg0/c171.dat b/metastore_db/seg0/c171.dat new file mode 100644 index 0000000..55d4095 Binary files /dev/null and b/metastore_db/seg0/c171.dat differ diff --git a/metastore_db/seg0/c180.dat b/metastore_db/seg0/c180.dat new file mode 100644 index 0000000..1d2ce89 Binary files /dev/null and b/metastore_db/seg0/c180.dat differ diff --git a/metastore_db/seg0/c191.dat b/metastore_db/seg0/c191.dat new file mode 100644 index 0000000..aa25052 Binary files /dev/null and b/metastore_db/seg0/c191.dat differ diff --git a/metastore_db/seg0/c1a1.dat b/metastore_db/seg0/c1a1.dat new file mode 100644 index 0000000..b459569 Binary files /dev/null and b/metastore_db/seg0/c1a1.dat differ diff --git a/metastore_db/seg0/c1b1.dat b/metastore_db/seg0/c1b1.dat new file mode 100644 index 0000000..eb94626 Binary files /dev/null and b/metastore_db/seg0/c1b1.dat differ diff --git a/metastore_db/seg0/c1c0.dat b/metastore_db/seg0/c1c0.dat new file mode 100644 index 0000000..c5b91e2 Binary files /dev/null and b/metastore_db/seg0/c1c0.dat differ diff --git a/metastore_db/seg0/c1d1.dat b/metastore_db/seg0/c1d1.dat new file mode 100644 index 0000000..451f02f Binary files /dev/null and b/metastore_db/seg0/c1d1.dat differ diff --git a/metastore_db/seg0/c1e0.dat b/metastore_db/seg0/c1e0.dat new file mode 100644 index 0000000..ac98556 Binary files /dev/null and b/metastore_db/seg0/c1e0.dat differ diff --git a/metastore_db/seg0/c1f1.dat b/metastore_db/seg0/c1f1.dat new file mode 100644 index 0000000..10629f1 Binary files /dev/null and b/metastore_db/seg0/c1f1.dat differ diff --git a/metastore_db/seg0/c20.dat b/metastore_db/seg0/c20.dat new file mode 100644 index 0000000..c55397e Binary files /dev/null and b/metastore_db/seg0/c20.dat differ diff --git a/metastore_db/seg0/c200.dat b/metastore_db/seg0/c200.dat new file mode 100644 index 0000000..b405709 Binary files /dev/null and b/metastore_db/seg0/c200.dat differ diff --git a/metastore_db/seg0/c211.dat b/metastore_db/seg0/c211.dat new file mode 100644 index 0000000..ae98a48 Binary files /dev/null and b/metastore_db/seg0/c211.dat differ diff --git a/metastore_db/seg0/c221.dat b/metastore_db/seg0/c221.dat new file mode 100644 index 0000000..a378dd4 Binary files /dev/null and b/metastore_db/seg0/c221.dat differ diff --git a/metastore_db/seg0/c230.dat b/metastore_db/seg0/c230.dat new file mode 100644 index 0000000..7e61b01 Binary files /dev/null and b/metastore_db/seg0/c230.dat differ diff --git a/metastore_db/seg0/c241.dat b/metastore_db/seg0/c241.dat new file mode 100644 index 0000000..1d7d21b Binary files /dev/null and b/metastore_db/seg0/c241.dat differ diff --git a/metastore_db/seg0/c251.dat b/metastore_db/seg0/c251.dat new file mode 100644 index 0000000..4b963f1 Binary files /dev/null and b/metastore_db/seg0/c251.dat differ diff --git a/metastore_db/seg0/c260.dat b/metastore_db/seg0/c260.dat new file mode 100644 index 0000000..25f81fd Binary files /dev/null and b/metastore_db/seg0/c260.dat differ diff --git a/metastore_db/seg0/c271.dat b/metastore_db/seg0/c271.dat new file mode 100644 index 0000000..51cde57 Binary files /dev/null and b/metastore_db/seg0/c271.dat differ diff --git a/metastore_db/seg0/c281.dat b/metastore_db/seg0/c281.dat new file mode 100644 index 0000000..cfed875 Binary files /dev/null and b/metastore_db/seg0/c281.dat differ diff --git a/metastore_db/seg0/c290.dat b/metastore_db/seg0/c290.dat new file mode 100644 index 0000000..4d10db4 Binary files /dev/null and b/metastore_db/seg0/c290.dat differ diff --git a/metastore_db/seg0/c2a1.dat b/metastore_db/seg0/c2a1.dat new file mode 100644 index 0000000..8e2ed6a Binary files /dev/null and b/metastore_db/seg0/c2a1.dat differ diff --git a/metastore_db/seg0/c2b1.dat b/metastore_db/seg0/c2b1.dat new file mode 100644 index 0000000..2a29692 Binary files /dev/null and b/metastore_db/seg0/c2b1.dat differ diff --git a/metastore_db/seg0/c2c1.dat b/metastore_db/seg0/c2c1.dat new file mode 100644 index 0000000..5511575 Binary files /dev/null and b/metastore_db/seg0/c2c1.dat differ diff --git a/metastore_db/seg0/c2d0.dat b/metastore_db/seg0/c2d0.dat new file mode 100644 index 0000000..4adc6e4 Binary files /dev/null and b/metastore_db/seg0/c2d0.dat differ diff --git a/metastore_db/seg0/c2e1.dat b/metastore_db/seg0/c2e1.dat new file mode 100644 index 0000000..b37b9b2 Binary files /dev/null and b/metastore_db/seg0/c2e1.dat differ diff --git a/metastore_db/seg0/c2f0.dat b/metastore_db/seg0/c2f0.dat new file mode 100644 index 0000000..d854b4b Binary files /dev/null and b/metastore_db/seg0/c2f0.dat differ diff --git a/metastore_db/seg0/c300.dat b/metastore_db/seg0/c300.dat new file mode 100644 index 0000000..2053e01 Binary files /dev/null and b/metastore_db/seg0/c300.dat differ diff --git a/metastore_db/seg0/c31.dat b/metastore_db/seg0/c31.dat new file mode 100644 index 0000000..f809dc7 Binary files /dev/null and b/metastore_db/seg0/c31.dat differ diff --git a/metastore_db/seg0/c311.dat b/metastore_db/seg0/c311.dat new file mode 100644 index 0000000..f60c260 Binary files /dev/null and b/metastore_db/seg0/c311.dat differ diff --git a/metastore_db/seg0/c321.dat b/metastore_db/seg0/c321.dat new file mode 100644 index 0000000..a9d7453 Binary files /dev/null and b/metastore_db/seg0/c321.dat differ diff --git a/metastore_db/seg0/c331.dat b/metastore_db/seg0/c331.dat new file mode 100644 index 0000000..85ee72b Binary files /dev/null and b/metastore_db/seg0/c331.dat differ diff --git a/metastore_db/seg0/c340.dat b/metastore_db/seg0/c340.dat new file mode 100644 index 0000000..d99b11a Binary files /dev/null and b/metastore_db/seg0/c340.dat differ diff --git a/metastore_db/seg0/c351.dat b/metastore_db/seg0/c351.dat new file mode 100644 index 0000000..f822f4c Binary files /dev/null and b/metastore_db/seg0/c351.dat differ diff --git a/metastore_db/seg0/c361.dat b/metastore_db/seg0/c361.dat new file mode 100644 index 0000000..b5c8f25 Binary files /dev/null and b/metastore_db/seg0/c361.dat differ diff --git a/metastore_db/seg0/c371.dat b/metastore_db/seg0/c371.dat new file mode 100644 index 0000000..ad11f01 Binary files /dev/null and b/metastore_db/seg0/c371.dat differ diff --git a/metastore_db/seg0/c380.dat b/metastore_db/seg0/c380.dat new file mode 100644 index 0000000..eebff34 Binary files /dev/null and b/metastore_db/seg0/c380.dat differ diff --git a/metastore_db/seg0/c391.dat b/metastore_db/seg0/c391.dat new file mode 100644 index 0000000..4373642 Binary files /dev/null and b/metastore_db/seg0/c391.dat differ diff --git a/metastore_db/seg0/c3a1.dat b/metastore_db/seg0/c3a1.dat new file mode 100644 index 0000000..f65ecdf Binary files /dev/null and b/metastore_db/seg0/c3a1.dat differ diff --git a/metastore_db/seg0/c3b1.dat b/metastore_db/seg0/c3b1.dat new file mode 100644 index 0000000..8f66060 Binary files /dev/null and b/metastore_db/seg0/c3b1.dat differ diff --git a/metastore_db/seg0/c3c0.dat b/metastore_db/seg0/c3c0.dat new file mode 100644 index 0000000..4d061cf Binary files /dev/null and b/metastore_db/seg0/c3c0.dat differ diff --git a/metastore_db/seg0/c3d1.dat b/metastore_db/seg0/c3d1.dat new file mode 100644 index 0000000..45c9fa2 Binary files /dev/null and b/metastore_db/seg0/c3d1.dat differ diff --git a/metastore_db/seg0/c3e1.dat b/metastore_db/seg0/c3e1.dat new file mode 100644 index 0000000..48f53e6 Binary files /dev/null and b/metastore_db/seg0/c3e1.dat differ diff --git a/metastore_db/seg0/c3f1.dat b/metastore_db/seg0/c3f1.dat new file mode 100644 index 0000000..08acdce Binary files /dev/null and b/metastore_db/seg0/c3f1.dat differ diff --git a/metastore_db/seg0/c400.dat b/metastore_db/seg0/c400.dat new file mode 100644 index 0000000..1e8976f Binary files /dev/null and b/metastore_db/seg0/c400.dat differ diff --git a/metastore_db/seg0/c41.dat b/metastore_db/seg0/c41.dat new file mode 100644 index 0000000..b16bdbf Binary files /dev/null and b/metastore_db/seg0/c41.dat differ diff --git a/metastore_db/seg0/c411.dat b/metastore_db/seg0/c411.dat new file mode 100644 index 0000000..8aba2fb Binary files /dev/null and b/metastore_db/seg0/c411.dat differ diff --git a/metastore_db/seg0/c421.dat b/metastore_db/seg0/c421.dat new file mode 100644 index 0000000..65775ee Binary files /dev/null and b/metastore_db/seg0/c421.dat differ diff --git a/metastore_db/seg0/c430.dat b/metastore_db/seg0/c430.dat new file mode 100644 index 0000000..55c948d Binary files /dev/null and b/metastore_db/seg0/c430.dat differ diff --git a/metastore_db/seg0/c441.dat b/metastore_db/seg0/c441.dat new file mode 100644 index 0000000..3948b2a Binary files /dev/null and b/metastore_db/seg0/c441.dat differ diff --git a/metastore_db/seg0/c451.dat b/metastore_db/seg0/c451.dat new file mode 100644 index 0000000..fe1ab73 Binary files /dev/null and b/metastore_db/seg0/c451.dat differ diff --git a/metastore_db/seg0/c461.dat b/metastore_db/seg0/c461.dat new file mode 100644 index 0000000..e6d9854 Binary files /dev/null and b/metastore_db/seg0/c461.dat differ diff --git a/metastore_db/seg0/c470.dat b/metastore_db/seg0/c470.dat new file mode 100644 index 0000000..c9f2eb1 Binary files /dev/null and b/metastore_db/seg0/c470.dat differ diff --git a/metastore_db/seg0/c481.dat b/metastore_db/seg0/c481.dat new file mode 100644 index 0000000..397b291 Binary files /dev/null and b/metastore_db/seg0/c481.dat differ diff --git a/metastore_db/seg0/c490.dat b/metastore_db/seg0/c490.dat new file mode 100644 index 0000000..2564a09 Binary files /dev/null and b/metastore_db/seg0/c490.dat differ diff --git a/metastore_db/seg0/c4a1.dat b/metastore_db/seg0/c4a1.dat new file mode 100644 index 0000000..563f8ff Binary files /dev/null and b/metastore_db/seg0/c4a1.dat differ diff --git a/metastore_db/seg0/c4b0.dat b/metastore_db/seg0/c4b0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c4b0.dat differ diff --git a/metastore_db/seg0/c4c1.dat b/metastore_db/seg0/c4c1.dat new file mode 100644 index 0000000..1d9bff1 Binary files /dev/null and b/metastore_db/seg0/c4c1.dat differ diff --git a/metastore_db/seg0/c4d1.dat b/metastore_db/seg0/c4d1.dat new file mode 100644 index 0000000..bcf5465 Binary files /dev/null and b/metastore_db/seg0/c4d1.dat differ diff --git a/metastore_db/seg0/c4e1.dat b/metastore_db/seg0/c4e1.dat new file mode 100644 index 0000000..0d71fdb Binary files /dev/null and b/metastore_db/seg0/c4e1.dat differ diff --git a/metastore_db/seg0/c4f0.dat b/metastore_db/seg0/c4f0.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c4f0.dat differ diff --git a/metastore_db/seg0/c501.dat b/metastore_db/seg0/c501.dat new file mode 100644 index 0000000..19429ff Binary files /dev/null and b/metastore_db/seg0/c501.dat differ diff --git a/metastore_db/seg0/c51.dat b/metastore_db/seg0/c51.dat new file mode 100644 index 0000000..55f3cc6 Binary files /dev/null and b/metastore_db/seg0/c51.dat differ diff --git a/metastore_db/seg0/c510.dat b/metastore_db/seg0/c510.dat new file mode 100644 index 0000000..2564a09 Binary files /dev/null and b/metastore_db/seg0/c510.dat differ diff --git a/metastore_db/seg0/c521.dat b/metastore_db/seg0/c521.dat new file mode 100644 index 0000000..12ec83f Binary files /dev/null and b/metastore_db/seg0/c521.dat differ diff --git a/metastore_db/seg0/c530.dat b/metastore_db/seg0/c530.dat new file mode 100644 index 0000000..2564a09 Binary files /dev/null and b/metastore_db/seg0/c530.dat differ diff --git a/metastore_db/seg0/c541.dat b/metastore_db/seg0/c541.dat new file mode 100644 index 0000000..baf7210 Binary files /dev/null and b/metastore_db/seg0/c541.dat differ diff --git a/metastore_db/seg0/c550.dat b/metastore_db/seg0/c550.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c550.dat differ diff --git a/metastore_db/seg0/c561.dat b/metastore_db/seg0/c561.dat new file mode 100644 index 0000000..338f98e Binary files /dev/null and b/metastore_db/seg0/c561.dat differ diff --git a/metastore_db/seg0/c570.dat b/metastore_db/seg0/c570.dat new file mode 100644 index 0000000..2564a09 Binary files /dev/null and b/metastore_db/seg0/c570.dat differ diff --git a/metastore_db/seg0/c581.dat b/metastore_db/seg0/c581.dat new file mode 100644 index 0000000..29d0eb5 Binary files /dev/null and b/metastore_db/seg0/c581.dat differ diff --git a/metastore_db/seg0/c590.dat b/metastore_db/seg0/c590.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c590.dat differ diff --git a/metastore_db/seg0/c5a1.dat b/metastore_db/seg0/c5a1.dat new file mode 100644 index 0000000..2a19eed Binary files /dev/null and b/metastore_db/seg0/c5a1.dat differ diff --git a/metastore_db/seg0/c5b0.dat b/metastore_db/seg0/c5b0.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c5b0.dat differ diff --git a/metastore_db/seg0/c5c1.dat b/metastore_db/seg0/c5c1.dat new file mode 100644 index 0000000..c55b726 Binary files /dev/null and b/metastore_db/seg0/c5c1.dat differ diff --git a/metastore_db/seg0/c5d0.dat b/metastore_db/seg0/c5d0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c5d0.dat differ diff --git a/metastore_db/seg0/c5e1.dat b/metastore_db/seg0/c5e1.dat new file mode 100644 index 0000000..c27c796 Binary files /dev/null and b/metastore_db/seg0/c5e1.dat differ diff --git a/metastore_db/seg0/c5f0.dat b/metastore_db/seg0/c5f0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c5f0.dat differ diff --git a/metastore_db/seg0/c60.dat b/metastore_db/seg0/c60.dat new file mode 100644 index 0000000..e8e794b Binary files /dev/null and b/metastore_db/seg0/c60.dat differ diff --git a/metastore_db/seg0/c601.dat b/metastore_db/seg0/c601.dat new file mode 100644 index 0000000..f06441d Binary files /dev/null and b/metastore_db/seg0/c601.dat differ diff --git a/metastore_db/seg0/c610.dat b/metastore_db/seg0/c610.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c610.dat differ diff --git a/metastore_db/seg0/c621.dat b/metastore_db/seg0/c621.dat new file mode 100644 index 0000000..3dfb417 Binary files /dev/null and b/metastore_db/seg0/c621.dat differ diff --git a/metastore_db/seg0/c630.dat b/metastore_db/seg0/c630.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c630.dat differ diff --git a/metastore_db/seg0/c641.dat b/metastore_db/seg0/c641.dat new file mode 100644 index 0000000..6393877 Binary files /dev/null and b/metastore_db/seg0/c641.dat differ diff --git a/metastore_db/seg0/c650.dat b/metastore_db/seg0/c650.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c650.dat differ diff --git a/metastore_db/seg0/c661.dat b/metastore_db/seg0/c661.dat new file mode 100644 index 0000000..88fae0a Binary files /dev/null and b/metastore_db/seg0/c661.dat differ diff --git a/metastore_db/seg0/c670.dat b/metastore_db/seg0/c670.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c670.dat differ diff --git a/metastore_db/seg0/c681.dat b/metastore_db/seg0/c681.dat new file mode 100644 index 0000000..1a592c3 Binary files /dev/null and b/metastore_db/seg0/c681.dat differ diff --git a/metastore_db/seg0/c690.dat b/metastore_db/seg0/c690.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c690.dat differ diff --git a/metastore_db/seg0/c6a1.dat b/metastore_db/seg0/c6a1.dat new file mode 100644 index 0000000..7aadc53 Binary files /dev/null and b/metastore_db/seg0/c6a1.dat differ diff --git a/metastore_db/seg0/c6b0.dat b/metastore_db/seg0/c6b0.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c6b0.dat differ diff --git a/metastore_db/seg0/c6c1.dat b/metastore_db/seg0/c6c1.dat new file mode 100644 index 0000000..586e530 Binary files /dev/null and b/metastore_db/seg0/c6c1.dat differ diff --git a/metastore_db/seg0/c6d0.dat b/metastore_db/seg0/c6d0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c6d0.dat differ diff --git a/metastore_db/seg0/c6e1.dat b/metastore_db/seg0/c6e1.dat new file mode 100644 index 0000000..efe6db9 Binary files /dev/null and b/metastore_db/seg0/c6e1.dat differ diff --git a/metastore_db/seg0/c6f0.dat b/metastore_db/seg0/c6f0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c6f0.dat differ diff --git a/metastore_db/seg0/c701.dat b/metastore_db/seg0/c701.dat new file mode 100644 index 0000000..ecbc088 Binary files /dev/null and b/metastore_db/seg0/c701.dat differ diff --git a/metastore_db/seg0/c71.dat b/metastore_db/seg0/c71.dat new file mode 100644 index 0000000..c0b1303 Binary files /dev/null and b/metastore_db/seg0/c71.dat differ diff --git a/metastore_db/seg0/c711.dat b/metastore_db/seg0/c711.dat new file mode 100644 index 0000000..cc99972 Binary files /dev/null and b/metastore_db/seg0/c711.dat differ diff --git a/metastore_db/seg0/c721.dat b/metastore_db/seg0/c721.dat new file mode 100644 index 0000000..256361d Binary files /dev/null and b/metastore_db/seg0/c721.dat differ diff --git a/metastore_db/seg0/c731.dat b/metastore_db/seg0/c731.dat new file mode 100644 index 0000000..3ae8d82 Binary files /dev/null and b/metastore_db/seg0/c731.dat differ diff --git a/metastore_db/seg0/c741.dat b/metastore_db/seg0/c741.dat new file mode 100644 index 0000000..c55242c Binary files /dev/null and b/metastore_db/seg0/c741.dat differ diff --git a/metastore_db/seg0/c751.dat b/metastore_db/seg0/c751.dat new file mode 100644 index 0000000..4a2797d Binary files /dev/null and b/metastore_db/seg0/c751.dat differ diff --git a/metastore_db/seg0/c761.dat b/metastore_db/seg0/c761.dat new file mode 100644 index 0000000..b871d2a Binary files /dev/null and b/metastore_db/seg0/c761.dat differ diff --git a/metastore_db/seg0/c771.dat b/metastore_db/seg0/c771.dat new file mode 100644 index 0000000..a9a996f Binary files /dev/null and b/metastore_db/seg0/c771.dat differ diff --git a/metastore_db/seg0/c781.dat b/metastore_db/seg0/c781.dat new file mode 100644 index 0000000..1a45028 Binary files /dev/null and b/metastore_db/seg0/c781.dat differ diff --git a/metastore_db/seg0/c791.dat b/metastore_db/seg0/c791.dat new file mode 100644 index 0000000..d392dcc Binary files /dev/null and b/metastore_db/seg0/c791.dat differ diff --git a/metastore_db/seg0/c7a1.dat b/metastore_db/seg0/c7a1.dat new file mode 100644 index 0000000..05e1d92 Binary files /dev/null and b/metastore_db/seg0/c7a1.dat differ diff --git a/metastore_db/seg0/c7b1.dat b/metastore_db/seg0/c7b1.dat new file mode 100644 index 0000000..f8827bf Binary files /dev/null and b/metastore_db/seg0/c7b1.dat differ diff --git a/metastore_db/seg0/c7c1.dat b/metastore_db/seg0/c7c1.dat new file mode 100644 index 0000000..1d393d5 Binary files /dev/null and b/metastore_db/seg0/c7c1.dat differ diff --git a/metastore_db/seg0/c7d1.dat b/metastore_db/seg0/c7d1.dat new file mode 100644 index 0000000..bee91c5 Binary files /dev/null and b/metastore_db/seg0/c7d1.dat differ diff --git a/metastore_db/seg0/c7e1.dat b/metastore_db/seg0/c7e1.dat new file mode 100644 index 0000000..7bd5c24 Binary files /dev/null and b/metastore_db/seg0/c7e1.dat differ diff --git a/metastore_db/seg0/c7f1.dat b/metastore_db/seg0/c7f1.dat new file mode 100644 index 0000000..28803f6 Binary files /dev/null and b/metastore_db/seg0/c7f1.dat differ diff --git a/metastore_db/seg0/c801.dat b/metastore_db/seg0/c801.dat new file mode 100644 index 0000000..ab99b96 Binary files /dev/null and b/metastore_db/seg0/c801.dat differ diff --git a/metastore_db/seg0/c81.dat b/metastore_db/seg0/c81.dat new file mode 100644 index 0000000..799f896 Binary files /dev/null and b/metastore_db/seg0/c81.dat differ diff --git a/metastore_db/seg0/c811.dat b/metastore_db/seg0/c811.dat new file mode 100644 index 0000000..9ef3d11 Binary files /dev/null and b/metastore_db/seg0/c811.dat differ diff --git a/metastore_db/seg0/c821.dat b/metastore_db/seg0/c821.dat new file mode 100644 index 0000000..5d5d6bf Binary files /dev/null and b/metastore_db/seg0/c821.dat differ diff --git a/metastore_db/seg0/c831.dat b/metastore_db/seg0/c831.dat new file mode 100644 index 0000000..88f3166 Binary files /dev/null and b/metastore_db/seg0/c831.dat differ diff --git a/metastore_db/seg0/c840.dat b/metastore_db/seg0/c840.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c840.dat differ diff --git a/metastore_db/seg0/c851.dat b/metastore_db/seg0/c851.dat new file mode 100644 index 0000000..274c69d Binary files /dev/null and b/metastore_db/seg0/c851.dat differ diff --git a/metastore_db/seg0/c860.dat b/metastore_db/seg0/c860.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c860.dat differ diff --git a/metastore_db/seg0/c871.dat b/metastore_db/seg0/c871.dat new file mode 100644 index 0000000..2e5b378 Binary files /dev/null and b/metastore_db/seg0/c871.dat differ diff --git a/metastore_db/seg0/c880.dat b/metastore_db/seg0/c880.dat new file mode 100644 index 0000000..6b1f5d8 Binary files /dev/null and b/metastore_db/seg0/c880.dat differ diff --git a/metastore_db/seg0/c891.dat b/metastore_db/seg0/c891.dat new file mode 100644 index 0000000..526a30a Binary files /dev/null and b/metastore_db/seg0/c891.dat differ diff --git a/metastore_db/seg0/c8a0.dat b/metastore_db/seg0/c8a0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c8a0.dat differ diff --git a/metastore_db/seg0/c8b1.dat b/metastore_db/seg0/c8b1.dat new file mode 100644 index 0000000..0bbd603 Binary files /dev/null and b/metastore_db/seg0/c8b1.dat differ diff --git a/metastore_db/seg0/c8c1.dat b/metastore_db/seg0/c8c1.dat new file mode 100644 index 0000000..59a500d Binary files /dev/null and b/metastore_db/seg0/c8c1.dat differ diff --git a/metastore_db/seg0/c8d1.dat b/metastore_db/seg0/c8d1.dat new file mode 100644 index 0000000..98c04ec Binary files /dev/null and b/metastore_db/seg0/c8d1.dat differ diff --git a/metastore_db/seg0/c8e1.dat b/metastore_db/seg0/c8e1.dat new file mode 100644 index 0000000..36cf238 Binary files /dev/null and b/metastore_db/seg0/c8e1.dat differ diff --git a/metastore_db/seg0/c8f1.dat b/metastore_db/seg0/c8f1.dat new file mode 100644 index 0000000..71f5d2b Binary files /dev/null and b/metastore_db/seg0/c8f1.dat differ diff --git a/metastore_db/seg0/c90.dat b/metastore_db/seg0/c90.dat new file mode 100644 index 0000000..d78be56 Binary files /dev/null and b/metastore_db/seg0/c90.dat differ diff --git a/metastore_db/seg0/c901.dat b/metastore_db/seg0/c901.dat new file mode 100644 index 0000000..679c965 Binary files /dev/null and b/metastore_db/seg0/c901.dat differ diff --git a/metastore_db/seg0/c911.dat b/metastore_db/seg0/c911.dat new file mode 100644 index 0000000..8b90e9e Binary files /dev/null and b/metastore_db/seg0/c911.dat differ diff --git a/metastore_db/seg0/c920.dat b/metastore_db/seg0/c920.dat new file mode 100644 index 0000000..f587783 Binary files /dev/null and b/metastore_db/seg0/c920.dat differ diff --git a/metastore_db/seg0/c931.dat b/metastore_db/seg0/c931.dat new file mode 100644 index 0000000..a7a309f Binary files /dev/null and b/metastore_db/seg0/c931.dat differ diff --git a/metastore_db/seg0/c940.dat b/metastore_db/seg0/c940.dat new file mode 100644 index 0000000..a685dc9 Binary files /dev/null and b/metastore_db/seg0/c940.dat differ diff --git a/metastore_db/seg0/c951.dat b/metastore_db/seg0/c951.dat new file mode 100644 index 0000000..2fe599d Binary files /dev/null and b/metastore_db/seg0/c951.dat differ diff --git a/metastore_db/seg0/c960.dat b/metastore_db/seg0/c960.dat new file mode 100644 index 0000000..dcceb54 Binary files /dev/null and b/metastore_db/seg0/c960.dat differ diff --git a/metastore_db/seg0/c971.dat b/metastore_db/seg0/c971.dat new file mode 100644 index 0000000..6bc4c35 Binary files /dev/null and b/metastore_db/seg0/c971.dat differ diff --git a/metastore_db/seg0/c981.dat b/metastore_db/seg0/c981.dat new file mode 100644 index 0000000..798399c Binary files /dev/null and b/metastore_db/seg0/c981.dat differ diff --git a/metastore_db/seg0/c990.dat b/metastore_db/seg0/c990.dat new file mode 100644 index 0000000..f587783 Binary files /dev/null and b/metastore_db/seg0/c990.dat differ diff --git a/metastore_db/seg0/c9a1.dat b/metastore_db/seg0/c9a1.dat new file mode 100644 index 0000000..f8138e9 Binary files /dev/null and b/metastore_db/seg0/c9a1.dat differ diff --git a/metastore_db/seg0/c9b1.dat b/metastore_db/seg0/c9b1.dat new file mode 100644 index 0000000..034194c Binary files /dev/null and b/metastore_db/seg0/c9b1.dat differ diff --git a/metastore_db/seg0/c9c0.dat b/metastore_db/seg0/c9c0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c9c0.dat differ diff --git a/metastore_db/seg0/c9d1.dat b/metastore_db/seg0/c9d1.dat new file mode 100644 index 0000000..088bde0 Binary files /dev/null and b/metastore_db/seg0/c9d1.dat differ diff --git a/metastore_db/seg0/c9e0.dat b/metastore_db/seg0/c9e0.dat new file mode 100644 index 0000000..4ad7351 Binary files /dev/null and b/metastore_db/seg0/c9e0.dat differ diff --git a/metastore_db/seg0/c9f1.dat b/metastore_db/seg0/c9f1.dat new file mode 100644 index 0000000..419b628 Binary files /dev/null and b/metastore_db/seg0/c9f1.dat differ diff --git a/metastore_db/seg0/ca01.dat b/metastore_db/seg0/ca01.dat new file mode 100644 index 0000000..3f8f81c Binary files /dev/null and b/metastore_db/seg0/ca01.dat differ diff --git a/metastore_db/seg0/ca1.dat b/metastore_db/seg0/ca1.dat new file mode 100644 index 0000000..605c9da Binary files /dev/null and b/metastore_db/seg0/ca1.dat differ diff --git a/metastore_db/seg0/ca11.dat b/metastore_db/seg0/ca11.dat new file mode 100644 index 0000000..f403aab Binary files /dev/null and b/metastore_db/seg0/ca11.dat differ diff --git a/metastore_db/seg0/ca21.dat b/metastore_db/seg0/ca21.dat new file mode 100644 index 0000000..ca5bad7 Binary files /dev/null and b/metastore_db/seg0/ca21.dat differ diff --git a/metastore_db/seg0/cb1.dat b/metastore_db/seg0/cb1.dat new file mode 100644 index 0000000..20f5882 Binary files /dev/null and b/metastore_db/seg0/cb1.dat differ diff --git a/metastore_db/seg0/cc0.dat b/metastore_db/seg0/cc0.dat new file mode 100644 index 0000000..2268720 Binary files /dev/null and b/metastore_db/seg0/cc0.dat differ diff --git a/metastore_db/seg0/cd1.dat b/metastore_db/seg0/cd1.dat new file mode 100644 index 0000000..d919a1b Binary files /dev/null and b/metastore_db/seg0/cd1.dat differ diff --git a/metastore_db/seg0/ce1.dat b/metastore_db/seg0/ce1.dat new file mode 100644 index 0000000..299e0c4 Binary files /dev/null and b/metastore_db/seg0/ce1.dat differ diff --git a/metastore_db/seg0/cf0.dat b/metastore_db/seg0/cf0.dat new file mode 100644 index 0000000..c355a5b Binary files /dev/null and b/metastore_db/seg0/cf0.dat differ diff --git a/metastore_db/service.properties b/metastore_db/service.properties new file mode 100644 index 0000000..28f40ee --- /dev/null +++ b/metastore_db/service.properties @@ -0,0 +1,23 @@ +#C:\Users\Administrator\IdeaProjects\spark\metastore_db +# ******************************************************************** +# *** Please do NOT edit this file. *** +# *** CHANGING THE CONTENT OF THIS FILE MAY CAUSE DATA CORRUPTION. *** +# ******************************************************************** +#Tue Dec 25 10:38:29 CST 2018 +SysschemasIndex2Identifier=225 +SyscolumnsIdentifier=144 +SysconglomeratesIndex1Identifier=49 +SysconglomeratesIdentifier=32 +SyscolumnsIndex2Identifier=177 +SysschemasIndex1Identifier=209 +SysconglomeratesIndex3Identifier=81 +SystablesIndex2Identifier=129 +SyscolumnsIndex1Identifier=161 +derby.serviceProtocol=org.apache.derby.database.Database +SysschemasIdentifier=192 +derby.storage.propertiesId=16 +SysconglomeratesIndex2Identifier=65 +derby.serviceLocale=zh_CN +SystablesIdentifier=96 +SystablesIndex1Identifier=113 +#--- last line, don't put anything after this line --- diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..c46b5d9 --- /dev/null +++ b/pom.xml @@ -0,0 +1,129 @@ + + + 4.0.0 + + com.itcast + spark + 1.0-SNAPSHOT + + + 2.11.8 + 2.7.4 + 2.0.2 + + + + org.scala-lang + scala-library + ${scala.version} + + + org.apache.spark + spark-core_2.11 + ${spark.version} + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + org.apache.spark + spark-sql_2.11 + 2.0.2 + + + org.apache.spark + spark-hive_2.11 + 2.0.2 + + + org.apache.hbase + hbase-client + 1.3.1 + + + mysql + mysql-connector-java + 5.1.30 + + + org.apache.kafka + kafka-clients + 0.10.1.1 + + + org.apache.spark + spark-streaming_2.11 + 2.0.2 + + + org.apache.spark + spark-streaming-flume_2.11 + 2.0.2 + + + org.apache.spark + spark-streaming-kafka-0-8_2.11 + 2.0.2 + + + + + src/main/scala + src/test/scala + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + + compile + testCompile + + + + -dependencyfile + ${project.build.directory}/.scala_dependencies + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/spark.iml b/spark.iml new file mode 100644 index 0000000..78b2cc5 --- /dev/null +++ b/spark.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/sparkStreamingCK/.checkpoint-1545909895000.bk.crc b/sparkStreamingCK/.checkpoint-1545909895000.bk.crc new file mode 100644 index 0000000..74f4ddc Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909895000.bk.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909895000.crc b/sparkStreamingCK/.checkpoint-1545909895000.crc new file mode 100644 index 0000000..2ec37e2 Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909895000.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909900000.bk.crc b/sparkStreamingCK/.checkpoint-1545909900000.bk.crc new file mode 100644 index 0000000..2073e58 Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909900000.bk.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909900000.crc b/sparkStreamingCK/.checkpoint-1545909900000.crc new file mode 100644 index 0000000..5c9b79f Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909900000.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909905000.bk.crc b/sparkStreamingCK/.checkpoint-1545909905000.bk.crc new file mode 100644 index 0000000..e19d66f Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909905000.bk.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909905000.crc b/sparkStreamingCK/.checkpoint-1545909905000.crc new file mode 100644 index 0000000..c5f025b Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909905000.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909910000.bk.crc b/sparkStreamingCK/.checkpoint-1545909910000.bk.crc new file mode 100644 index 0000000..703e3bc Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909910000.bk.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909910000.crc b/sparkStreamingCK/.checkpoint-1545909910000.crc new file mode 100644 index 0000000..6854329 Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909910000.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909915000.bk.crc b/sparkStreamingCK/.checkpoint-1545909915000.bk.crc new file mode 100644 index 0000000..2d13095 Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909915000.bk.crc differ diff --git a/sparkStreamingCK/.checkpoint-1545909915000.crc b/sparkStreamingCK/.checkpoint-1545909915000.crc new file mode 100644 index 0000000..3b98af6 Binary files /dev/null and b/sparkStreamingCK/.checkpoint-1545909915000.crc differ diff --git a/sparkStreamingCK/checkpoint-1545909895000 b/sparkStreamingCK/checkpoint-1545909895000 new file mode 100644 index 0000000..34bf46a Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909895000 differ diff --git a/sparkStreamingCK/checkpoint-1545909895000.bk b/sparkStreamingCK/checkpoint-1545909895000.bk new file mode 100644 index 0000000..b3f3603 Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909895000.bk differ diff --git a/sparkStreamingCK/checkpoint-1545909900000 b/sparkStreamingCK/checkpoint-1545909900000 new file mode 100644 index 0000000..8dcd356 Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909900000 differ diff --git a/sparkStreamingCK/checkpoint-1545909900000.bk b/sparkStreamingCK/checkpoint-1545909900000.bk new file mode 100644 index 0000000..e6a619a Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909900000.bk differ diff --git a/sparkStreamingCK/checkpoint-1545909905000 b/sparkStreamingCK/checkpoint-1545909905000 new file mode 100644 index 0000000..9f4c20e Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909905000 differ diff --git a/sparkStreamingCK/checkpoint-1545909905000.bk b/sparkStreamingCK/checkpoint-1545909905000.bk new file mode 100644 index 0000000..2c9a34a Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909905000.bk differ diff --git a/sparkStreamingCK/checkpoint-1545909910000 b/sparkStreamingCK/checkpoint-1545909910000 new file mode 100644 index 0000000..069ee65 Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909910000 differ diff --git a/sparkStreamingCK/checkpoint-1545909910000.bk b/sparkStreamingCK/checkpoint-1545909910000.bk new file mode 100644 index 0000000..c56292e Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909910000.bk differ diff --git a/sparkStreamingCK/checkpoint-1545909915000 b/sparkStreamingCK/checkpoint-1545909915000 new file mode 100644 index 0000000..2373c07 Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909915000 differ diff --git a/sparkStreamingCK/checkpoint-1545909915000.bk b/sparkStreamingCK/checkpoint-1545909915000.bk new file mode 100644 index 0000000..6918438 Binary files /dev/null and b/sparkStreamingCK/checkpoint-1545909915000.bk differ diff --git a/sparkStreamingCK/receivedBlockMetadata/log-1545909885001-1545909945001 b/sparkStreamingCK/receivedBlockMetadata/log-1545909885001-1545909945001 new file mode 100644 index 0000000..0904a81 Binary files /dev/null and b/sparkStreamingCK/receivedBlockMetadata/log-1545909885001-1545909945001 differ diff --git a/sparkStreamingCK/receivedData/0/log-1545908708005-1545908768005 b/sparkStreamingCK/receivedData/0/log-1545908708005-1545908768005 new file mode 100644 index 0000000..9718238 Binary files /dev/null and b/sparkStreamingCK/receivedData/0/log-1545908708005-1545908768005 differ diff --git a/src/main/java/com/itheima/hbase/TestHbase.java b/src/main/java/com/itheima/hbase/TestHbase.java new file mode 100644 index 0000000..7a05d5b --- /dev/null +++ b/src/main/java/com/itheima/hbase/TestHbase.java @@ -0,0 +1,148 @@ +package com.itheima.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.util.Bytes; + +import java.util.List; + + +public class TestHbase { + public static void main(String[] args) throws Exception { + //HBase的两种连接方式： + // 1读取配置文件 只需要配置zookeeper + /*Configuration config = HBaseConfiguration.create(); + Connection connection = ConnectionFactory.createConnection(config);*/ + + //2 通过代码配置 每次需要改代码不建议 + /*Configuration configuration = new Configuration(); + configuration.set("hbase.zookeeper.quorum", "node1:2181,node2:2181,node3:2181"); + connection = ConnectionFactory.createConnection();*/ + + // 1.连接HBase + // 1.1 HBaseConfiguration.create(); + Configuration config = HBaseConfiguration.create(); + // 1.2 创建一个连接 + Connection connection = ConnectionFactory.createConnection(config); + // 1.3 从连接中获得一个Admin对象 + Admin admin = connection.getAdmin(); + // 2.创建表 + // 2.1 判断表是否存在 + TableName tableName = TableName.valueOf("user"); + if (!admin.tableExists(tableName)) { + //2.2 如果表不存在就创建一个表 + HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName); + hTableDescriptor.addFamily(new HColumnDescriptor("base_info")); + admin.createTable(hTableDescriptor); + System.out.println("创建表"); + } + + System.out.println("-------------------打印表的描述信息------------------------"); + // 1.定义表的名称 + TableName tableName1 = TableName.valueOf("user"); + // 2.获取表 + Table table1 = connection.getTable(tableName1); + // 3.获取表的描述信息 + HTableDescriptor tableDescriptor = table1.getTableDescriptor(); + // 4.获取表的列簇信息 + HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies(); + for (HColumnDescriptor columnFamily : columnFamilies) { + // 5.获取表的columFamily的字节数组 + byte[] name = columnFamily.getName(); + // 6.使用hbase自带的bytes工具类转成string + String value = Bytes.toString(name); + // 7.打印 + System.out.println(value); + } + + System.out.println("-------------------添加数据(PUT)------------------------"); + // 1.定义表的名称 + TableName tableName2 = TableName.valueOf("user"); + // 2.获取表 + Table table2 = connection.getTable(tableName2); + // 3.准备数据 + String rowKey2 = "rowkey_10"; + Put zhangsan = new Put(Bytes.toBytes(rowKey2)); + zhangsan.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("张三")); + zhangsan.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("sex"), Bytes.toBytes("1")); + zhangsan.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("address"), Bytes.toBytes("北京市")); + zhangsan.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("birthday"), Bytes.toBytes("2014-07-10")); + // 4. 添加数据 + table2.put(zhangsan); + table2.close(); + + System.out.println("-------------------获取数据(Get)------------------------"); + // 1.定义表的名称 + TableName tableName3 = TableName.valueOf("user"); + // 2.获取表 + Table table3 = connection.getTable(tableName3); + // 3.准备数据 + String rowKey3 = "rowkey_10"; + // 4.拼装查询条件 + Get get = new Get(Bytes.toBytes(rowKey3)); + // 5.查询数据 + Result result = table3.get(get); + // 6.打印数据 获取所有的单元格 + List cells = result.listCells(); + for (Cell cell : cells) { + // 打印rowkey,family,qualifier,value + System.out.println(Bytes.toString(CellUtil.cloneRow(cell)) + + "==> " + Bytes.toString(CellUtil.cloneFamily(cell)) + + "{" + Bytes.toString(CellUtil.cloneQualifier(cell)) + + ":" + Bytes.toString(CellUtil.cloneValue(cell)) + "}"); + } + + System.out.println("-------------------全盘扫描(Scan)-----------------------"); + // 1.定义表的名称 + TableName tableName4 = TableName.valueOf("user"); + // 2.获取表 + Table table4 = connection.getTable(tableName4); + // 3.全表扫描 + Scan scan = new Scan(); + // 定义上下界 + scan.setStartRow(Bytes.toBytes("rowkey_10")); + scan.setStopRow(Bytes.toBytes("rowkey_20")); + // 4.获取扫描结果 + ResultScanner scanner = table4.getScanner(scan); + Result result4 = null; + // 5. 迭代数据 + while ((result4 = scanner.next()) != null) { + // 6.打印数据 获取所有的单元格 + List cells4 = result4.listCells(); + for (Cell cell : cells4) { + // 打印rowkey,family,qualifier,value + System.out.println(Bytes.toString(CellUtil.cloneRow(cell)) + + "==> " + Bytes.toString(CellUtil.cloneFamily(cell)) + + "{" + Bytes.toString(CellUtil.cloneQualifier(cell)) + + ":" + Bytes.toString(CellUtil.cloneValue(cell)) + "}"); + } + } + + System.out.println("-------------------过滤查询-----------------------"); + //1、创建过滤器 + ValueFilter filter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator("张三".getBytes())); + //2、创建扫描器 + Scan scan5 = new Scan(); + //3、将过滤器设置到扫描器中 + scan5.setFilter(filter); + //4、获取HBase的表 + Table table5 = connection.getTable(TableName.valueOf("user")); + //5、扫描HBase的表（注意过滤操作是在服务器进行的，也即是在regionServer进行的） + ResultScanner scanner5 = table5.getScanner(scan5); + for (Result result5 : scanner5) { + // 6.打印数据 获取所有的单元格 + List cells5 = result5.listCells(); + for (Cell cell : cells5) { + // 打印rowkey,family,qualifier,value + System.out.println(Bytes.toString(CellUtil.cloneRow(cell)) + + "==> " + Bytes.toString(CellUtil.cloneFamily(cell)) + + "{" + Bytes.toString(CellUtil.cloneQualifier(cell)) + + ":" + Bytes.toString(CellUtil.cloneValue(cell)) + "}"); + } + } + } +} diff --git a/src/main/java/com/itheima/kafka/ConsumerDemo.java b/src/main/java/com/itheima/kafka/ConsumerDemo.java new file mode 100644 index 0000000..cf84544 --- /dev/null +++ b/src/main/java/com/itheima/kafka/ConsumerDemo.java @@ -0,0 +1,34 @@ +package com.itheima.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; + +import java.util.Arrays; +import java.util.Properties; + +public class ConsumerDemo { + public static void main(String[] args) { + //1、配置信息 + Properties props = new Properties(); + props.put("bootstrap.servers", "node1:9092,node2:9092,node3:9092"); + props.put("group.id", "test"); + props.put("enable.auto.commit", "true"); + props.put("auto.commit.interval.ms", "1000"); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + + //2 创建kafka消费对象 + KafkaConsumer consumer = new KafkaConsumer(props); + + //3 消费具体kafka消费对象 + consumer.subscribe(Arrays.asList("itcast")); + + //4、遍历获取来自于具体的topic数据 + while (true) { + ConsumerRecords records = consumer.poll(100); + for (ConsumerRecord record : records) + System.out.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value()); + } + } +} diff --git a/src/main/java/com/itheima/kafka/DataFromConsumer2Producer.java b/src/main/java/com/itheima/kafka/DataFromConsumer2Producer.java new file mode 100644 index 0000000..3c2c9c0 --- /dev/null +++ b/src/main/java/com/itheima/kafka/DataFromConsumer2Producer.java @@ -0,0 +1,76 @@ +package com.itheima.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; + +import java.util.Arrays; +import java.util.Properties; + +public class DataFromConsumer2Producer { + public static void main(String[] args) { + //kafkaConsumer + //1、配置信息 + Properties props = new Properties(); + props.put("bootstrap.servers", "node1:9092,node2:9092,node3:9092"); + props.put("group.id", "test"); + props.put("enable.auto.commit", "true"); + props.put("auto.commit.interval.ms", "1000"); + props.put("session.timeout.ms", "30000"); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + + //2、创建kafka消费者对象 + KafkaConsumer consumer = new KafkaConsumer(props); + + //3、消费具体topic中的数据 + consumer.subscribe(Arrays.asList("itcast")); + + String msg = ""; + while (true) { + + //4、遍历获取来自于具体的topic数据 + ConsumerRecords records = consumer.poll(100); + for (ConsumerRecord record : records) { + msg = record.value(); + sendMsg(msg); + + System.out.println(msg); + } + } + + } + + static Properties props = null; + + //生产者对象的配置-只需要加载一次 + static { + //1、准备配置项-KafkaProducer创建的时候，必须的一些配置项 + props = new Properties(); + props.put("bootstrap.servers", "node1:9092,node2:9092,node3:9092"); + props.put("acks", "all"); + props.put("retries", 0); + props.put("batch.size", 16384); + props.put("linger.ms", 1); + props.put("buffer.memory", 33554432); + props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + } + + + /** + * 生产数据 + * @param msg + */ + public static void sendMsg(String msg) { + //2、创建生产者对象 -KafkaProducer + Producer producer = new KafkaProducer(props); + //3、发送数据到具体的topic + producer.send(new ProducerRecord("TOPIC_Heima", "", msg)); + //4、释放资源 + producer.close(); + } +} diff --git a/src/main/java/com/itheima/kafka/ProducerDemo.java b/src/main/java/com/itheima/kafka/ProducerDemo.java new file mode 100644 index 0000000..5c59ed1 --- /dev/null +++ b/src/main/java/com/itheima/kafka/ProducerDemo.java @@ -0,0 +1,43 @@ +package com.itheima.kafka; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; + +import java.util.Properties; + +public class ProducerDemo { + public static void main(String[] args) { + //1、准备配置项-KafkaProducer创建的时候，必须的一些配置项 + Properties props = new Properties(); + props.put("bootstrap.servers", "node1:9092,node2:9092,node3:9092"); + props.put("acks", "all"); + props.put("retries", 0); + props.put("batch.size", 16384); + props.put("linger.ms", 1); + props.put("buffer.memory", 33554432); + props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + + //2、创建生产者对象 -KafkaProducer + Producer producer = new KafkaProducer(props); + for(int i = 0; i < 100; i++){ + + //3、发送数据到具体的topic + producer.send(new ProducerRecord("itcast", "", Integer.toString(i)+"hadoop")); + + + //1、数据分发策略：没有指定具体的分区，也没有key---->走 round-robin 轮询策略进行分发数据 + producer.send(new ProducerRecord("itcast","hello")); + + //2、有key的这种形式，数据分发策略为：基于key的hash + producer.send(new ProducerRecord("itcast","key","value")); + + //3、数据分发策略-指定了具体的分区，就将数据分发到指定的分区里面去了---不建议使用 + producer.send(new ProducerRecord("itcast",2,"key","value")); + } + + //4、释放资源 + producer.close(); + } +} diff --git a/src/main/resources/hbase-site.xml b/src/main/resources/hbase-site.xml new file mode 100644 index 0000000..aaa8cb9 --- /dev/null +++ b/src/main/resources/hbase-site.xml @@ -0,0 +1,9 @@ + + + + + hbase.zookeeper.quorum + node1,node2,node3 + The directory shared by region servers. + + \ No newline at end of file diff --git a/src/main/scala/demo/demo1.scala b/src/main/scala/demo/demo1.scala new file mode 100644 index 0000000..fcba86f --- /dev/null +++ b/src/main/scala/demo/demo1.scala @@ -0,0 +1,31 @@ +package demo + +import org.apache.spark.rdd.RDD +import org.apache.spark.{SparkConf, SparkContext} + +object demo1 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo1").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + //1 通过已经存在的集合进行创建 + val rdd1: RDD[Int] = sc.parallelize(Array(1,2,3,4,5,6,7)) + + //2 由外部存储系统的文件创建 + val rdd2: RDD[String] = sc.textFile("C:\\hadoop\\test\\input\\inputword\\1.txt") + + //3 已有的RDD经过算子转换成新的RDD + val rdd3: RDD[String] = rdd2.flatMap(_.split(" ")) + + val rdd4 = rdd3.map((_,1)) + + val count = rdd3.count() + val first = rdd3.first() + val take = rdd3.take(10) + val sorted = rdd4.sortByKey(false) + + rdd1.foreach(x=>print(x)) + rdd3.foreach(x=>println(x)) + } +} diff --git a/src/main/scala/demo/demo2.scala b/src/main/scala/demo/demo2.scala new file mode 100644 index 0000000..98f1e13 --- /dev/null +++ b/src/main/scala/demo/demo2.scala @@ -0,0 +1,142 @@ +package demo + +import org.apache.spark.{SparkConf, SparkContext} + +object demo2_1 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + val rdd1 = sc.parallelize(List(5, 6, 4, 7, 3, 8, 2, 9, 1, 10)) + val rdd2 = rdd1.map(_ * 2).sortBy(x => x, true) + val rdd3 = rdd2.filter(_ >= 5) + println(rdd3.collect().toBuffer) + } +} + +object demo2_2 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + val rdd1 = sc.parallelize(Array("a b c", "d e f", "h i j")) + val rdd2 = rdd1.flatMap(_.split(" ")) + println(rdd2.collect().toBuffer) + } +} + +object demo2_3 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + val rdd1 = sc.parallelize(Array(5, 6, 4, 3)) + val rdd2 = sc.parallelize(Array(1, 2, 3, 4)) + + //并集 + val rdd3 = rdd1.union(rdd2) + //交集 + val rdd4 = rdd1.intersection(rdd2) + //去重 + val rdd5 = rdd3.distinct + + println(rdd4.collect().toBuffer) + println(rdd5.collect().toBuffer) + } +} + +object demo2_4 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + val rdd1 = sc.parallelize(List(("tom", 1), ("jerry", 3), ("kitty", 2))) + val rdd2 = sc.parallelize(List(("jerry", 2), ("tom", 1), ("shuke", 2))) + + //join + val rdd3 = rdd1.join(rdd2) + //并集 + val rdd4 = rdd1.union(rdd2) + //去重 + val rdd5 = rdd4.groupByKey() + + val rdd6 = rdd4.groupBy(_._1).mapValues(_.foldLeft(0)(_ + _._2)) + println(rdd6.collect().toBuffer) + } +} + +object demo2_5 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + val rdd1 = sc.parallelize(List(("tom", 1), ("tom", 2), ("jerry", 3), ("kitty", 2))) + val rdd2 = sc.parallelize(List(("jerry", 2), ("tom", 1), ("jim", 2))) + + //cogroup + val rdd3 = rdd1.cogroup(rdd2) + val rdd4 = rdd3.collect.map(x => (x._1, x._2._1.sum + x._2._2.sum)) + println(rdd4.toBuffer) + } +} + +object demo2_6 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + val rdd1 = sc.parallelize(List(1, 2, 3, 4, 5)) + + //reduce + val rdd2 = rdd1.reduce(_ + _) + println(rdd2) + } +} + +object demo2_7 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + val rdd1 = sc.parallelize(List(("tom", 1), ("jerry", 3), ("kitty", 2), ("shuke", 1))) + val rdd2 = sc.parallelize(List(("jerry", 2), ("tom", 3), ("shuke", 2), ("kitty", 5))) + val rdd3 = rdd1.union(rdd2) + + //普通解法 + val result = rdd3.groupBy(_._1).map(x=>(x._1,x._2.map(_._2).sum)) + println(result.collect.toBuffer) + + //reduce解法 + val rdd4 = rdd3.reduceByKey(_ + _) + println(rdd4.collect.toBuffer) + + val rdd5 = rdd4.map(t => (t._2, t._1)).sortByKey(false).map(t => (t._2, t._1)) + println(rdd5.collect.toBuffer) + } +} + +object demo2_8 { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("demo2").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + val rdd1 = sc.parallelize(1 to 10,3) + + //利用repartition改变rdd1分区数 + //减少分区 + rdd1.repartition(2).partitions.size + //增加分区 + rdd1.repartition(4).partitions.size + + //利用coalesce改变rdd1分区数 只能减不能加 + //减少分区 + rdd1.coalesce(2).partitions.size + } +} diff --git a/src/main/scala/rdd/IpLocation.scala b/src/main/scala/rdd/IpLocation.scala new file mode 100644 index 0000000..65fc0c2 --- /dev/null +++ b/src/main/scala/rdd/IpLocation.scala @@ -0,0 +1,118 @@ +package rdd + +import java.sql.{Connection, DriverManager, PreparedStatement} + +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.rdd.RDD +import org.apache.spark.{SparkConf, SparkContext} + +object IpLocation { + + def ipToLong(ip: String): Long = { + val split: Array[String] = ip.split("\\.") + var ipNum = 0L + for (i <- split) { + ipNum = i.toLong | ipNum << 8L + } + ipNum + } + + def binarySearch(ip: Long, valueArr: Array[(String, String, String, String)]): Int = { + + var start = 0 + var end = valueArr.length - 1 + + while (start <= end) { + var middle = (start + end) / 2 + + if (ip >= valueArr(middle)._1.toLong && ip <= valueArr(middle)._2.toLong) { + return middle + } + + if (ip < valueArr(middle)._1.toLong) { + end = middle + } + + if (ip > valueArr(middle)._2.toLong) { + start = middle + } + } + -1 + } + + + def data2Mysql(iterator: Iterator[(String, String, Int)]): Unit = { + + var conn: Connection = null + + var ps: PreparedStatement = null + + var sql = "insert into iplocation(longitude,latitude,total_count) values(?,?,?)" + + conn = DriverManager.getConnection("jdbc:mysql://node1:3306/itcast", "root", "hadoop") + + try { + iterator.foreach(line => { + + ps = conn.prepareStatement(sql) + + ps.setString(1, line._1) + ps.setString(2, line._2) + ps.setLong(3, line._3) + + ps.execute() + }) + } catch { + case e: Exception => println(e) + } finally { + if (ps != null) { + ps.close() + } + if (conn != null) { + conn.close() + } + } + } + + def main(args: Array[String]): Unit = { + + val sparkConf: SparkConf = new SparkConf().setAppName("IpLocation").setMaster("local[8]") + + val sc: SparkContext = new SparkContext(sparkConf) + + sc.setLogLevel("WARN") + + val ips: RDD[String] = sc.textFile("C:\\hadoop\\test\\ip.txt") + + val map: RDD[(String, String, String, String)] = ips.map(_.split("\\|")).map(x => (x(2), x(3), x(13), x(14))) + + val collect: Array[(String, String, String, String)] = map.collect() + + val broadcast: Broadcast[Array[(String, String, String, String)]] = sc.broadcast(collect) + + val destData: RDD[String] = sc.textFile("C:\\hadoop\\test\\20090121000132.394251.http.format") + + val map1: RDD[Array[String]] = destData.map(_.split("\\|")) + + val ipAddr: RDD[String] = map1.map(x => x(1)) + + val partitions: RDD[((String, String), Int)] = ipAddr.mapPartitions(inter => { + val valueArr: Array[(String, String, String, String)] = broadcast.value + inter.map(ip => { + val ipNum: Long = ipToLong(ip) + val index: Int = binarySearch(ipNum, valueArr) + val arr: (String, String, String, String) = valueArr(index) + ((arr._3, arr._4), 1) + }) + }) + val key: RDD[((String, String), Int)] = partitions.reduceByKey(_ + _) + + key.foreach(print) + + key.map(x => (x._1._1, x._1._2, x._2)).foreachPartition(data2Mysql) + + sc.stop() + + + } +} diff --git a/src/main/scala/rdd/PV.scala b/src/main/scala/rdd/PV.scala new file mode 100644 index 0000000..c8e0361 --- /dev/null +++ b/src/main/scala/rdd/PV.scala @@ -0,0 +1,23 @@ +package rdd + +import org.apache.spark.{SparkConf, SparkContext} + +object PV { + def main(args: Array[String]): Unit = { + //需求:利用spark分析点击流日志数据---PV总量 + //1 创建SparkConf对象 + val sparkConf = new SparkConf().setAppName("PV").setMaster("local[2]") + + //2 创建SparkContext对象 + val sc = new SparkContext(sparkConf) + + //3 读取数据文件 + val data = sc.textFile("C:\\hadoop\\test\\access.log") + + //4 获取PV总量 + println(data.count()) + + //5 关闭sparkContext + sc.stop() + } +} diff --git a/src/main/scala/rdd/Test.scala b/src/main/scala/rdd/Test.scala new file mode 100644 index 0000000..a425e41 --- /dev/null +++ b/src/main/scala/rdd/Test.scala @@ -0,0 +1,19 @@ +package rdd + +object Test { + def main(args: Array[String]): Unit = { + val ip="1.3.0.0" + println(ip2Long((ip))) + } + + //将ip地址转化为Long 192.168.200.100 + def ip2Long(ip: String): Long = { + //将IP地址转为Long,这里有固定的算法 + val ips: Array[String] = ip.split("\\.") + var ipNum:Long=0L + for(i <- ips){ + ipNum=i.toLong | ipNum <<8L + } + ipNum + } +} diff --git a/src/main/scala/rdd/TopN.scala b/src/main/scala/rdd/TopN.scala new file mode 100644 index 0000000..d086465 --- /dev/null +++ b/src/main/scala/rdd/TopN.scala @@ -0,0 +1,35 @@ +package rdd + +import org.apache.spark.rdd.RDD +import org.apache.spark.{SparkConf, SparkContext} + +object TopN { + def main(args: Array[String]): Unit = { + //1 创建SparkConf对象 + val sparkConf = new SparkConf().setAppName("PV").setMaster("local[2]") + + //2 创建SparkContext对象 + val sc = new SparkContext(sparkConf) + + //3 读取数据文件 + val data = sc.textFile("C:\\hadoop\\test\\access.log") + + //4 切分每一行,获取IP,把每个IP记为1 + val IPAndOne = data.map(x => (x.split(" ")(0), 1)) + + //5 聚合操作 + val result = IPAndOne.reduceByKey(_ + _) + + //6 排序--倒序 + val sortData: RDD[(String, Int)] = result.sortBy(_._2,false) + + //7 取出前5个 + val finalResult: Array[(String, Int)] = sortData.take(5) + + //8 打印 + println(finalResult.toBuffer)//sortData.collect.foreach(x => println(x)) 或 print(sortData.collect.toBuffer) + + //9 关闭sparkContext + sc.stop() + } +} diff --git a/src/main/scala/rdd/UV.scala b/src/main/scala/rdd/UV.scala new file mode 100644 index 0000000..3960c15 --- /dev/null +++ b/src/main/scala/rdd/UV.scala @@ -0,0 +1,28 @@ +package rdd + +import org.apache.spark.{SparkConf, SparkContext} + +object UV { + def main(args: Array[String]): Unit = { + //1 创建SparkConf对象 + val sparkConf = new SparkConf().setAppName("UV").setMaster("local[2]") + + //2 创建SparkContext对象 + val sc = new SparkContext(sparkConf) + + //3 读取数据文件 + val data = sc.textFile("C:\\hadoop\\test\\access.log") + + //4 切分每一行获取ip地址 + val ips = data.map(_.split(" ")(0)) + + //5 获取UV总量 + val distinctIps = ips.distinct() + + //6 获取UV总量 + println(distinctIps.count()) + + //7 关闭sparkContext + sc.stop() + } +} diff --git a/src/main/scala/sparkStreaming/AcceptSocketDataForWC.scala b/src/main/scala/sparkStreaming/AcceptSocketDataForWC.scala new file mode 100644 index 0000000..ec0d4ee --- /dev/null +++ b/src/main/scala/sparkStreaming/AcceptSocketDataForWC.scala @@ -0,0 +1,42 @@ +package sparkStreaming + +import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream} +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.{SparkConf, SparkContext} + +/** + * 需求：接收socket的数据，进行单词计数 + * + */ +object AcceptSocketDataForWC { + def main(args: Array[String]): Unit = { + //1 创建一个对象 + val sparkConf = new SparkConf().setAppName("AcceptSocketDataForWC").setMaster("local[4]") + + //2 创建SparkContext对象 + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + //3 创建一个StreamingContext对象 + val ssc: StreamingContext = new StreamingContext(sc,Seconds(5)) + + //4 接收socket的数据 通过socketTextStream来获取数据 + val lines: ReceiverInputDStream[String] = ssc.socketTextStream("node1",9999) + + //5 单词计数 + val words: DStream[String] = lines.flatMap(_.split(" ")) + val wordAnd1 = words.map((_,1)) + + //一组相同key的数据进行累加 + val result = wordAnd1.reduceByKey(_+_) + + //6 对计算结果进行输出 + result.print() + + //启动流式计算 + ssc.start() + + //让程序一直执行 + ssc.awaitTermination() + } +} diff --git a/src/main/scala/sparkStreaming/AcceptSocketDataForWCTotal.scala b/src/main/scala/sparkStreaming/AcceptSocketDataForWCTotal.scala new file mode 100644 index 0000000..bd93402 --- /dev/null +++ b/src/main/scala/sparkStreaming/AcceptSocketDataForWCTotal.scala @@ -0,0 +1,55 @@ +package sparkStreaming + +import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream} +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.{SparkConf, SparkContext} + +/** + * 需求：接收socket数据 进行全局汇总的单词计数 + */ +object AcceptSocketDataForWCTotal { + def updateFunc(newValue: Seq[Int], runningValue: Option[Int]): Option[Int] = { + + val result: Int = newValue.sum + runningValue.getOrElse(0) + + Some(result) + } + + def updateFunc1(currentV: Seq[Int], historyV: Option[Int]): Option[Int] = { + + val result: Int = currentV.sum +historyV.getOrElse(0) + + Some(result) + } + + def main(args: Array[String]): Unit = { + //1 创建一个sparkConf对象 + val sparkConf: SparkConf = new SparkConf().setAppName("AcceptSocketDataForWCTotal").setMaster("local[4]") + + //2 创建sparkContext对象 + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + //3 创建一个streamingContext对象 + val ssc: StreamingContext = new StreamingContext(sc,Seconds(5)) + + //需要记录历史数据进行统计 就要把历史数据保存下来 必须要指定一个路径 + ssc.checkpoint("./ck") + + //4 接收socket的数据 通过socketTextStream来获取数据 + val lines: ReceiverInputDStream[String] = ssc.socketTextStream("node1",9999) + + //5 单词计数 + val result: DStream[(String, Int)] = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_) + val historyResult: DStream[(String, Int)] = result.updateStateByKey(updateFunc) + + //6 对计算的结果进行输出 + historyResult.print() + + //启动流式计算 + ssc.start() + + //让程序一直进行 + ssc.awaitTermination() + } +} diff --git a/src/main/scala/sparkStreaming/FlumePoll.scala b/src/main/scala/sparkStreaming/FlumePoll.scala new file mode 100644 index 0000000..f579cc4 --- /dev/null +++ b/src/main/scala/sparkStreaming/FlumePoll.scala @@ -0,0 +1,44 @@ +package sparkStreaming + +import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream} +import org.apache.spark.streaming.flume.{FlumeUtils, SparkFlumeEvent} +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.{SparkConf, SparkContext} + +/** + * 需求：通过sparkStreaming来拉取flume中的数据，然后进行单词计数 + * + * 难点：怎么样拉取函数入口？ + * FlumeUtils + * + * 特点： + * 1、sparkStreaming程序主动的去flume中拉取数据 + * 2、 + */ +object FlumePoll { + def main(args: Array[String]): Unit = { + //1 初始化工作 + val sparkConf = new SparkConf().setAppName("FlumePoll").setMaster("local[4]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + //2 创建StreamingContext + val ssc = new StreamingContext(sc,Seconds(5)) + + //3 通过sparkStreaming和flume的整合包中有一个入口FlumeUtils + val stream: ReceiverInputDStream[SparkFlumeEvent] = FlumeUtils.createPollingStream(ssc,"node1",8888) + + //4 将具体的数据拿出来 数据在body中 + val lines: DStream[String] = stream.map(x=>new String(x.event.getBody.array())) + + //5 进行单词计数 + val result: DStream[(String, Int)] = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_) + + //6 输出结果 + result.print() + + //7 启动程序 + ssc.start() + ssc.awaitTermination() + } +} diff --git a/src/main/scala/sparkStreaming/FlumePush.scala b/src/main/scala/sparkStreaming/FlumePush.scala new file mode 100644 index 0000000..8390f91 --- /dev/null +++ b/src/main/scala/sparkStreaming/FlumePush.scala @@ -0,0 +1,34 @@ +package sparkStreaming + +import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream} +import org.apache.spark.streaming.flume.{FlumeUtils, SparkFlumeEvent} +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.{SparkConf, SparkContext} + +object FlumePush { + def main(args: Array[String]): Unit = { + //1 初始化操作 + val sparkConf: SparkConf = new SparkConf().setAppName("FlumePush").setMaster("local[4]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + //2 创建ssc对象 + val ssc = new StreamingContext(sc,Seconds(5)) + + //3 接收flume push过来的数据 + val stream: ReceiverInputDStream[SparkFlumeEvent] = FlumeUtils.createStream(ssc,"192.168.155.58",9999) + + //4 读取数据 + val lines: DStream[String] = stream.map(x=>new String(x.event.getBody.array())) + + //5 业务逻辑处理 单词计数 + val result: DStream[(String, Int)] = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_) + + //6 获取数据 进行打印 + result.print() + + //7 开启流式计算 + ssc.start() + ssc.awaitTermination() + } +} diff --git a/src/main/scala/sparkStreaming/KafkaCreateDStream.scala b/src/main/scala/sparkStreaming/KafkaCreateDStream.scala new file mode 100644 index 0000000..adf80cc --- /dev/null +++ b/src/main/scala/sparkStreaming/KafkaCreateDStream.scala @@ -0,0 +1,48 @@ +package sparkStreaming + +import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream} +import org.apache.spark.streaming.kafka.KafkaUtils +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.{SparkConf, SparkContext} + +/** + * 需求：通过sparkStreaming来拉取kafka中的数据---进行业务处理--进行单词计数 + */ +object KafkaCreateDStream { + def main(args: Array[String]): Unit = { + //1 初始化操作 + val sparkConf = new SparkConf().setAppName("KafkaCreateDStream") + .set("spark.streaming.receiver.writeAheadLog.enable", "true")//开启WAL预写日志 + .setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + val ssc = new StreamingContext(sc, Seconds(5)) + + //只要开启了writeAheadLog 就需要设置chkPoint目录 + ssc.checkpoint("./sparkStreamingCK") + + //zk config para + val zkQuorum = "node1:2181,node2:2181,node3:2181" + //kafka group name + val groupId = "group-itcast" + + //指定了topic名字 + val topics = Map("itcast" -> 2) + + //2 拉取数据 + val stream: ReceiverInputDStream[(String, String)] = KafkaUtils.createStream(ssc, zkQuorum, groupId, topics) + + //读取kafka中的数据：数据存储在ReceiverInputDStream[(String, String)]的第二位 + val lines: DStream[String] = stream.map(_._2) + + //进行具体的业务逻辑 + val result: DStream[(String, Int)] = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _) + + //打印计算结果 + result.print() + + //启动流式计算 + ssc.start() + ssc.awaitTermination() + } +} diff --git a/src/main/scala/sparkStreaming/SparkStreamingTOPN.scala b/src/main/scala/sparkStreaming/SparkStreamingTOPN.scala new file mode 100644 index 0000000..5980ad9 --- /dev/null +++ b/src/main/scala/sparkStreaming/SparkStreamingTOPN.scala @@ -0,0 +1,50 @@ +package sparkStreaming + +import org.apache.spark.rdd.RDD +import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream} +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.{SparkConf, SparkContext} + +/** + * 需求：通过开窗函数来统计一定时间间隔内的热门词汇---topN + * + * 关键点：需要在开窗函数的结果中，要进行排序-降序 + */ +object SparkStreamingTOPN { + def main(args: Array[String]): Unit = { + //1：初始化操作 + val sparkConf: SparkConf = new SparkConf().setAppName("SparkStreamingTOPN").setMaster("local[2]") + val sc: SparkContext = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + //2、sparkStreaming的入口 + val ssc: StreamingContext = new StreamingContext(sc, Seconds(5)) + + //3、接收数据 + val lines: ReceiverInputDStream[String] = ssc.socketTextStream("node1", 9999) + + //4、数据转换成元组(单词，1) + val wordAnd1: DStream[(String, Int)] = lines.flatMap(_.split(" ")).map((_, 1)) + + //5 使用开窗函数进行计算 + val result: DStream[(String, Int)] = wordAnd1.reduceByKeyAndWindow((a: Int, b: Int) => a + b, Seconds(5), Seconds(5)) + + //6 对当前时间范围之内的数据进行排序 取top3 + val data: DStream[(String, Int)] = result.transform(rdd => { + val dataRDD: RDD[(String, Int)] = rdd.sortBy(_._2, false) + val hotWords = dataRDD.take(3) + println("-----------start-----------") + + println(hotWords.toBuffer) + + println("-----------end-----------") + dataRDD + }) + + data.print() + + //启动流式计算 + ssc.start() + ssc.awaitTermination() + } +} diff --git a/src/main/scala/sparkStreaming/SparkStreamingWindowFunc.scala b/src/main/scala/sparkStreaming/SparkStreamingWindowFunc.scala new file mode 100644 index 0000000..dfecb57 --- /dev/null +++ b/src/main/scala/sparkStreaming/SparkStreamingWindowFunc.scala @@ -0,0 +1,42 @@ +package sparkStreaming + +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream} +import org.apache.spark.{SparkConf, SparkContext} + +/** + * 需求：通过reduceByKeyAndWindow来实现单词技术 + * 两个参数需要注意： + * 1、窗口大小 + * 2、滑动的时间 + * + * 如果参数没有设计好，会出现数据的重复计算和数据丢失 + */ +object SparkStreamingWindowFunc { + def main(args: Array[String]): Unit = { + //1 初始化操作 + val sparkConf = new SparkConf().setAppName("SparkStreamingWindowFunc").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + + //2 sparkStreaming函数的入口对象 + val ssc: StreamingContext = new StreamingContext(sc,Seconds(5)) + + //3 通过StreamingContext对象来获取socket数据 + val lines: ReceiverInputDStream[String] = ssc.socketTextStream("node1",9999) + + //4 将单次计数返回结果 两个注意的参数：窗口的长度，滑动窗口的时间 (注意：最好是创建StreamingContext批次时间的整数倍) + val window = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKeyAndWindow((a: Int, b: Int) => a + b, Seconds(10), Seconds(10)) + //val window = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKeyAndWindow((a: Int, b: Int) => a + b, Seconds(5), Seconds(5))正常 + //val window = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKeyAndWindow((a: Int, b: Int) => a + b, Seconds(5), Seconds(10))丢数据 + //val window = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKeyAndWindow((a: Int, b: Int) => a + b, Seconds(10), Seconds(5))重复计算数据 + + //output opera + window.print() + + //启动程序 + ssc.start() + ssc.awaitTermination() + } + +} diff --git a/src/main/scala/sparkStreaming/kafkaCreateDirectDStream.scala b/src/main/scala/sparkStreaming/kafkaCreateDirectDStream.scala new file mode 100644 index 0000000..b87cf3e --- /dev/null +++ b/src/main/scala/sparkStreaming/kafkaCreateDirectDStream.scala @@ -0,0 +1,39 @@ +package sparkStreaming + +import kafka.serializer.StringDecoder +import org.apache.spark.streaming.dstream.{DStream, InputDStream} +import org.apache.spark.streaming.kafka.KafkaUtils +import org.apache.spark.streaming.{Seconds, StreamingContext} +import org.apache.spark.{SparkConf, SparkContext} + +object kafkaCreateDirectDStream { + def main(args: Array[String]): Unit = { + //1 初始化操作 + val sparkConf = new SparkConf().setAppName("kafkaCreateDirectDStream").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + val ssc = new StreamingContext(sc, Seconds(5)) + + //kafka参数配置 + val kafkaParams = Map("metadata.broker.list" -> "node1:9092,node2:9092,node3:9092", "group.id" -> "Kafka_Direct") + + //配置topic参数 + val topics = Set("itcast") + + //2 获取kafka中的数据 + val stream: InputDStream[(String, String)] = KafkaUtils.createDirectStream[String,String,StringDecoder,StringDecoder](ssc, kafkaParams, topics) + + //获取InputStream中的值 + val lines: DStream[String] = stream.map(_._2) + + //具体业务 + val result: DStream[(String, Int)] = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _) + + //执行outPut operations + result.print() + + //开启流式计算 + ssc.start() + ssc.awaitTermination() + } +} diff --git a/src/main/scala/sql/DynamicDemo.scala b/src/main/scala/sql/DynamicDemo.scala new file mode 100644 index 0000000..c1e6c2c --- /dev/null +++ b/src/main/scala/sql/DynamicDemo.scala @@ -0,0 +1,36 @@ +package sql + +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} +import org.apache.spark.sql.{Row, SparkSession} + +object DynamicDemo { + def main(args: Array[String]): Unit = { + //初始化操作 + val spark: SparkSession = SparkSession.builder().appName("DynamicDemo").master("local[2]").getOrCreate() + val sc: SparkContext = spark.sparkContext + + sc.setLogLevel("WARN") + + //加载数据 + val lines: RDD[String] = sc.textFile("C:\\hadoop\\test\\Person.txt") + + //切分数据 + val words: RDD[Array[String]] = lines.map(_.split(" ")) + + //将数据转换成RDD[Row] + val rowRDD: RDD[Row] = words.map(x => Row(x(0).toInt, x(1), x(2).toInt)) + + val schema = StructType(StructField("id", IntegerType, false) :: StructField("name", StringType, false) :: StructField("age", IntegerType, false) :: Nil) + + ///创建dataframe + val dataFrame = spark.createDataFrame(rowRDD,schema) + dataFrame.show() + dataFrame.printSchema() + + //释放资源 + sc.stop() + spark.stop() + } +} diff --git a/src/main/scala/sql/JDBCDemo.scala b/src/main/scala/sql/JDBCDemo.scala new file mode 100644 index 0000000..187fef1 --- /dev/null +++ b/src/main/scala/sql/JDBCDemo.scala @@ -0,0 +1,32 @@ +package sql + +import java.util.Properties + +import org.apache.spark.SparkContext +import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession} + +object JDBCDemo { + def main(args: Array[String]): Unit = { + //初始化操作 + val spark = SparkSession.builder().appName("JDBCDemo").master("local[2]").getOrCreate() + val sc: SparkContext = spark.sparkContext + sc.setLogLevel("WARN") + + //配置登陆数据库的名字 密码 + val properties: Properties = new Properties() + properties.setProperty("user","root") + properties.setProperty("password","hadoop") + + //通过spark.read.jdbc来读取数据库的信息 返回结果为DataFrame + val dataframe: DataFrame = spark.read.jdbc("jdbc:mysql://node1:3306/itcast","iplocation",properties) + dataframe.show() + dataframe.printSchema() + + //写入的数据是什么 write.jdbc + //dataframe.write.mode(SaveMode.Overwrite).jdbc("jdbc:mysql://node1:3306/itcast","iplocation111",properties) + dataframe.write.mode(SaveMode.Append).jdbc("jdbc:mysql://node1:3306/itcast","iplocation111",properties) + + sc.stop() + spark.stop() + } +} diff --git a/src/main/scala/sql/OperaHive.scala b/src/main/scala/sql/OperaHive.scala new file mode 100644 index 0000000..c947c33 --- /dev/null +++ b/src/main/scala/sql/OperaHive.scala @@ -0,0 +1,24 @@ +package sql + +import org.apache.spark.SparkContext +import org.apache.spark.sql.SparkSession + +object OperaHive { + def main(args: Array[String]): Unit = { + //初始化操作 + val spark: SparkSession = SparkSession.builder().appName("OperaHive").master("local[2]") + .enableHiveSupport()//操作hive必须要打开操作hive的开关 + .config("spark.sql.warehouse.dir","C:\\hadoop\\test\\spark-warehouse") + .getOrCreate() + val sc: SparkContext = spark.sparkContext + + sc.setLogLevel("WARN") + + spark.sql("create table if not exists person(id int,name string, age int) row format delimited fields terminated by ' '") + spark.sql("load data local inpath './data/student.txt' into table person") + spark.sql("select * from person").show() + + sc.stop() + spark.stop() + } +} diff --git a/src/main/scala/sql/ReflectDemo.scala b/src/main/scala/sql/ReflectDemo.scala new file mode 100644 index 0000000..71bd4a2 --- /dev/null +++ b/src/main/scala/sql/ReflectDemo.scala @@ -0,0 +1,49 @@ +package sql + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{DataFrame, SparkSession} + +object ReflectDemo { + + //需求：通过反射来操作DataFrame + + //创建一个样例类与数据格式相对应 + case class Person(id: Int, name: String, age: Int) + + def main(args: Array[String]): Unit = { + //初始化操 + val spark: SparkSession = SparkSession.builder().appName("ReflectDemo").master("local[2]").getOrCreate() + + val sc = spark.sparkContext + + //过滤日志 + sc.setLogLevel("WARN") + + //加载数据 + val lines: RDD[String] = sc.textFile("C:\\hadoop\\test\\Person.txt") + + //切分数据 + val words: RDD[Array[String]] = lines.map(_.split(" ")) + + //将数据与样例类进行关联 + val personRDD: RDD[Person] = words.map(x=>Person(x(0).toInt,x(1),x(2).toInt)) + + //将RDD转换为DataFrame + import spark.implicits._ + val dataframe: DataFrame = personRDD.toDF + + //DataFrame + dataframe.show() + dataframe.printSchema() + dataframe.select("age","name").show() + dataframe.filter($"age">25).show() + + //通过sql方式来写的时候，一定要注册为一张表 + dataframe.createOrReplaceTempView("person") + spark.sql("select * from person where age >25").show() + sc.stop() + spark.stop() + + } + +} diff --git a/src/main/scala/wordcount/WordCount.scala b/src/main/scala/wordcount/WordCount.scala new file mode 100644 index 0000000..3fd5fd9 --- /dev/null +++ b/src/main/scala/wordcount/WordCount.scala @@ -0,0 +1,38 @@ +package wordcount + +import org.apache.spark.rdd.RDD +import org.apache.spark.{SparkConf, SparkContext} + +object WordCount { + def main(args: Array[String]): Unit = { + //1 创建sparkConfig对象 设置appName和master的地址 + val sparkConf: SparkConf = new SparkConf().setAppName("WordCount") + + //2 创建sparkContext对象 + val sc = new SparkContext(sparkConf) + + //设置日志输出级别 + sc.setLogLevel("WARN") + + //3 读取数据文件 + val data: RDD[String] = sc.textFile(args(0)) + + //4 切分文件中的每一行 返回文件所有的单词 + val words: RDD[String] = data.flatMap(_.split(" ")) + + //5 每个单词记为1 + val wordAndOne: RDD[(String, Int)] = words.map((_,1)) + + //6 相同单词出现的次数累加 + val result: RDD[(String, Int)] = wordAndOne.reduceByKey(_+_) + + //按照单词出现的次数降序排列 + val sortResult: RDD[(String, Int)] = result.sortBy(_._2,false) + + //7 按照数据保存在HDFS上 + sortResult.saveAsTextFile(args(1)) + + //8 关闭sc + sc.stop() + } +} diff --git a/src/main/scala/wordcount/WordCountLocal.scala b/src/main/scala/wordcount/WordCountLocal.scala new file mode 100644 index 0000000..06ea9b8 --- /dev/null +++ b/src/main/scala/wordcount/WordCountLocal.scala @@ -0,0 +1,38 @@ +package wordcount + +import org.apache.spark.{SparkConf, SparkContext} + +object WordCountLocal { + def main(args: Array[String]): Unit = { + val sparkConf = new SparkConf().setAppName("WordCount").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + sc.setLogLevel("WARN") + val data = sc.textFile("C:\\hadoop\\test\\input\\inputword\\1.txt") + val words = data.flatMap(_.split(" ")) + val wordAndOne = words.map((_,1)) + val result = wordAndOne.reduceByKey(_+_) + val sortResult = result.sortBy(_._2,false) + val finalResult = sortResult.collect() + finalResult.foreach(x=>println(x)) + sc.stop() + } +} + +object Test { + def main(args:Array[String]):Unit = { + val sparkConf = new SparkConf().setAppName("WordCount").setMaster("local[2]") + val sc = new SparkContext(sparkConf) + val lst = List(1,2,3,4,5); + print("foreach遍历：") + lst.foreach { x => print(x+",")} //foreach遍历,这个是传统遍历，新手不熟无奈之下可以用它 + println("") + + val rdd = sc.parallelize(List(1,2,3,5,8)) + var counter = 10 + //warn: don't do this + val unit: Unit = rdd.foreach(x => counter += x) + println("Counter value: "+counter) + + sc.stop() + } +}