-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathspark-operator.yaml
68 lines (68 loc) · 1.75 KB
/
spark-operator.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
#
# See: https://raw.githubusercontent.com/GoogleCloudPlatform/spark-on-k8s-operator/master/examples/spark-pi-prometheus.yaml
#
apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
name: gcs-example-operator
namespace: default
spec:
type: Scala
mode: cluster
image: "gcr.io/spark-operator/spark:v2.4.4-gcs-prometheus"
imagePullPolicy: Always
mainClass: yourMainClass
mainApplicationFile: "gs://sparknetes/fraud-joiner-assembly-1.1.jar"
arguments:
- "gs://fraud-dataset/dataset/bkOn"
- "gs://sparknetes/fraud.parquet"
hadoopConf:
"fs.gs.project.id": "sparknetes"
"fs.gs.system.bucket": "sparknetes"
"google.cloud.auth.service.account.enable": "true"
"google.cloud.auth.service.account.json.keyfile": "/mnt/secrets/key.json"
restartPolicy:
type: Never
volumes:
- name: "test-volume"
hostPath:
path: "/tmp"
type: Directory
driver:
cores: 2
memory: "2048m"
labels:
version: 2.4.4
sparknetes: "true"
serviceAccount: spark
volumeMounts:
- name: "test-volume"
mountPath: "/tmp"
secrets:
- name: gcloud-creds
path: /mnt/secrets
secretType: GCPServiceAccount
envVars:
GCS_PROJECT_ID: sparknetes
executor:
cores: 2
instances: 2
memory: "2048m"
labels:
version: 2.4.4
sparknetes: "true"
volumeMounts:
- name: "test-volume"
mountPath: "/tmp"
secrets:
- name: gcloud-creds
path: /mnt/secrets
secretType: GCPServiceAccount
envVars:
GCS_PROJECT_ID: sparknetes
monitoring:
exposeDriverMetrics: true
exposeExecutorMetrics: true
prometheus:
jmxExporterJar: "/prometheus/jmx_prometheus_javaagent-0.3.1.jar"
port: 8090