Skip to content

Commit 1858e57

Browse files
authoredOct 17, 2018
Update updataData.py
1 parent 733a1ef commit 1858e57

File tree

1 file changed

+12
-12
lines changed

1 file changed

+12
-12
lines changed
 

‎updataData.py

+12-12
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
1-
reader =spark.read.format("org.elasticsearch.spark.sql").option("es.read.metadata", "true").option("es.read.field.as.array.include", "title") \
1+
from pyspark import SparkContext, SparkConf
2+
from pyspark.sql import SQLContext
3+
4+
reader =spark.read.format("org.elasticsearch.spark.sql") \
5+
.option("es.read.metadata", "true") \
6+
.option("es.read.field.as.array.include", "title") \
27
.option("es.read.field.as.array.include", "outlinks_unique") \
3-
.option("es.nodes.wan.only","true").option("es.port","9200").option("es.net.ssl","false").option("es.nodes", "http://localhost")
8+
.option("es.nodes.wan.only","true") \
9+
.option("es.port","9200") \
10+
.option("es.net.ssl","false") \
11+
.option("es.nodes", "http://localhost")
412

513
df = reader.load("schools")
614

715
df.filter(df["school"] == "Harvard").show()
816

9-
j = {
10-
"script": {
11-
"lang": "painless",
12-
"inline": "ctx._source.location = params.location",
13-
"params": location:<Boston>
14-
}
15-
}
16-
1717
esconf={}
1818
esconf["es.mapping.id" = 1 ]
1919
esconf["es.nodes"] = "localhost"
2020
esconf["es.port"] = "9200"
21-
esconf["es.update.script.inline"] = j
22-
esconf["es.update.script.params"] = location:<Cambridge>
21+
esconf["es.update.script.inline"] = "ctx._source.location = params.location"
22+
esconf["es.update.script.params"] = "location:<Cambridge>"
2323
esconf["es.write.operation"] = "update"
2424

2525
df.write.format("org.elasticsearch.spark.sql").options(**esconf).mode("overwrite").save("backup_/items")

0 commit comments

Comments
 (0)
Please sign in to comment.