forked from andkret/Cookbook
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Data Engineering Cookbook.aux
353 lines (353 loc) · 30.8 KB
/
Data Engineering Cookbook.aux
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
\relax
\providecommand\hyper@newdestlabel[2]{}
\providecommand*\new@tpo@label[2]{}
\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument}
\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined
\global\let\oldcontentsline\contentsline
\gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}}
\global\let\oldnewlabel\newlabel
\gdef\newlabel#1#2{\newlabelxx{#1}#2}
\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}}
\AtEndDocument{\ifx\hyper@anchor\@undefined
\let\contentsline\oldcontentsline
\let\newlabel\oldnewlabel
\fi}
\fi}
\global\let\hyper@last\relax
\gdef\HyperFirstAtBeginDocument#1{#1}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\@writefile{toc}{\contentsline {part}{\numberline {I}Introduction}{9}{part.1}}
\@writefile{toc}{\contentsline {chapter}{\numberline {1}How To Use This Cookbook}{10}{chapter.1}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {chapter}{\numberline {2}Data Engineer vs Data Scientists}{11}{chapter.2}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {2.1}Data Scientist}{11}{section.2.1}}
\@writefile{lof}{\contentsline {figure}{\numberline {2.1}{\ignorespaces The Machine Learning Pipeline}}{12}{figure.2.1}}
\newlabel{fig:Bild1}{{2.1}{12}{The Machine Learning Pipeline}{figure.2.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {2.2}Data Engineer}{12}{section.2.2}}
\@writefile{toc}{\contentsline {section}{\numberline {2.3}Who Companies Need}{13}{section.2.3}}
\@writefile{toc}{\contentsline {part}{\numberline {II}Basic Data Engineering Skills}{14}{part.2}}
\@writefile{toc}{\contentsline {chapter}{\numberline {3}Learn To Code}{15}{chapter.3}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {chapter}{\numberline {4}Get Familiar With Github}{16}{chapter.4}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {chapter}{\numberline {5}Agile Development -- available}{17}{chapter.5}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {5.1}Why is agile so important?}{17}{section.5.1}}
\@writefile{toc}{\contentsline {section}{\numberline {5.2}Agile rules I learned over the years -- available}{18}{section.5.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2.1}Is the method making a difference?}{18}{subsection.5.2.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2.2}The problem with outsourcing}{18}{subsection.5.2.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2.3}Knowledge is king: A lesson from Elon Musk}{19}{subsection.5.2.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2.4}How you really can be agile}{19}{subsection.5.2.4}}
\@writefile{toc}{\contentsline {section}{\numberline {5.3}Agile Techniques}{20}{section.5.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.3.1}Scrum}{20}{subsection.5.3.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.3.2}OKR}{20}{subsection.5.3.2}}
\@writefile{toc}{\contentsline {chapter}{\numberline {6}Learn how a Computer Works}{21}{chapter.6}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {6.1}CPU,RAM,GPU,HDD}{21}{section.6.1}}
\@writefile{toc}{\contentsline {section}{\numberline {6.2}Differences between PCs and Servers}{21}{section.6.2}}
\@writefile{toc}{\contentsline {chapter}{\numberline {7}Computer Networking - Data Transmission}{22}{chapter.7}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {7.1}ISO/OSI Model}{22}{section.7.1}}
\@writefile{toc}{\contentsline {section}{\numberline {7.2}IP Subnetting}{22}{section.7.2}}
\@writefile{toc}{\contentsline {section}{\numberline {7.3}Switch, Level 3 Switch}{22}{section.7.3}}
\@writefile{toc}{\contentsline {section}{\numberline {7.4}Router}{22}{section.7.4}}
\@writefile{toc}{\contentsline {section}{\numberline {7.5}Firewalls}{22}{section.7.5}}
\@writefile{toc}{\contentsline {chapter}{\numberline {8}Security and Privacy}{23}{chapter.8}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {8.1}SSL Public \& Private Key Certificates}{23}{section.8.1}}
\@writefile{toc}{\contentsline {section}{\numberline {8.2}What is a certificate authority}{23}{section.8.2}}
\@writefile{toc}{\contentsline {section}{\numberline {8.3}JAva Web Tokens}{23}{section.8.3}}
\@writefile{toc}{\contentsline {section}{\numberline {8.4}GDPR regulations}{23}{section.8.4}}
\@writefile{toc}{\contentsline {section}{\numberline {8.5}Privacy by design}{23}{section.8.5}}
\@writefile{toc}{\contentsline {chapter}{\numberline {9}Linux}{24}{chapter.9}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {9.1}OS Basics}{24}{section.9.1}}
\@writefile{toc}{\contentsline {section}{\numberline {9.2}Shell scripting}{24}{section.9.2}}
\@writefile{toc}{\contentsline {section}{\numberline {9.3}Cron jobs}{24}{section.9.3}}
\@writefile{toc}{\contentsline {section}{\numberline {9.4}Packet management}{24}{section.9.4}}
\@writefile{toc}{\contentsline {chapter}{\numberline {10}The Cloud}{25}{chapter.10}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {10.1}AWS,Azure, IBM, Google Cloud basics}{25}{section.10.1}}
\@writefile{toc}{\contentsline {section}{\numberline {10.2}cloud vs on premise}{25}{section.10.2}}
\@writefile{toc}{\contentsline {section}{\numberline {10.3}up \& downsides}{25}{section.10.3}}
\@writefile{toc}{\contentsline {section}{\numberline {10.4}Security}{25}{section.10.4}}
\@writefile{toc}{\contentsline {chapter}{\numberline {11}Security Zone Design}{26}{chapter.11}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {11.1}How to secure a multi layered application}{26}{section.11.1}}
\@writefile{toc}{\contentsline {section}{\numberline {11.2}Cluster security with Kerberos}{26}{section.11.2}}
\@writefile{toc}{\contentsline {section}{\numberline {11.3}Kerberos Tickets}{26}{section.11.3}}
\@writefile{toc}{\contentsline {chapter}{\numberline {12}Stream Processing}{27}{chapter.12}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {12.1}Three methods of streaming | available}{27}{section.12.1}}
\@writefile{toc}{\contentsline {section}{\numberline {12.2}At Least Once}{27}{section.12.2}}
\@writefile{toc}{\contentsline {section}{\numberline {12.3}At Most Once}{28}{section.12.3}}
\@writefile{toc}{\contentsline {section}{\numberline {12.4}Exactly Once}{28}{section.12.4}}
\@writefile{toc}{\contentsline {section}{\numberline {12.5}Check The Tools!}{28}{section.12.5}}
\@writefile{toc}{\contentsline {chapter}{\numberline {13}Big Data}{29}{chapter.13}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {13.1}What is big data and where is the difference to data science and data analytics?}{29}{section.13.1}}
\@writefile{toc}{\contentsline {section}{\numberline {13.2}The 4Vs of Big Data | available}{29}{section.13.2}}
\@writefile{toc}{\contentsline {section}{\numberline {13.3}Why Big Data? | available}{30}{section.13.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.1}Planning is Everything}{31}{subsection.13.3.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.2}The Problem With ETL}{31}{subsection.13.3.2}}
\@writefile{lof}{\contentsline {figure}{\numberline {13.1}{\ignorespaces Common SQL Platform Architecture}}{31}{figure.13.1}}
\newlabel{fig:Bild1}{{13.1}{31}{Common SQL Platform Architecture}{figure.13.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.3}Scaling Up}{32}{subsection.13.3.3}}
\@writefile{lof}{\contentsline {figure}{\numberline {13.2}{\ignorespaces Scaling up a SQL Database}}{32}{figure.13.2}}
\newlabel{fig:Bild1}{{13.2}{32}{Scaling up a SQL Database}{figure.13.2}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.4}Scaling Out}{33}{subsection.13.3.4}}
\@writefile{lof}{\contentsline {figure}{\numberline {13.3}{\ignorespaces Scaling out a SQL Database}}{33}{figure.13.3}}
\newlabel{fig:Bild1}{{13.3}{33}{Scaling out a SQL Database}{figure.13.3}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.5}Please Don\IeC {\textquoteright }t go Big Data}{34}{subsection.13.3.5}}
\@writefile{toc}{\contentsline {chapter}{\numberline {14}Data Warehouse vs Data Lake}{35}{chapter.14}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {chapter}{\numberline {15}Hadoop Platforms | available}{36}{chapter.15}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {15.1}What is Hadoop}{36}{section.15.1}}
\@writefile{toc}{\contentsline {section}{\numberline {15.2}What makes Hadoop so popular? | available}{36}{section.15.2}}
\@writefile{lof}{\contentsline {figure}{\numberline {15.1}{\ignorespaces Hadoop Ecosystem Components}}{37}{figure.15.1}}
\newlabel{fig:Bild1}{{15.1}{37}{Hadoop Ecosystem Components}{figure.15.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {15.3}Hadoop Ecosystem Components}{37}{section.15.3}}
\@writefile{lof}{\contentsline {figure}{\numberline {15.2}{\ignorespaces Connections between tools}}{38}{figure.15.2}}
\newlabel{fig:Bild1}{{15.2}{38}{Connections between tools}{figure.15.2}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {15.3}{\ignorespaces Flume Integration}}{39}{figure.15.3}}
\newlabel{fig:Bild1}{{15.3}{39}{Flume Integration}{figure.15.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {15.4}Hadoop Is Everywhere?}{39}{section.15.4}}
\@writefile{toc}{\contentsline {section}{\numberline {15.5}SHOULD YOU LEARN HADOOP?}{40}{section.15.5}}
\@writefile{toc}{\contentsline {subsubsection}{\nonumberline How does a Hadoop System architecture look like}{40}{section*.2}}
\@writefile{toc}{\contentsline {subsubsection}{\nonumberline What tools are usually in a with Hadoop Cluster}{40}{section*.3}}
\@writefile{toc}{\contentsline {section}{\numberline {15.6}How to select Hadoop Cluster Hardware}{40}{section.15.6}}
\@writefile{toc}{\contentsline {chapter}{\numberline {16}Is ETL still relevant for Analytics?}{41}{chapter.16}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {chapter}{\numberline {17}Docker}{42}{chapter.17}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {17.1}What is docker and what do you use it for | available}{42}{section.17.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {17.1.1}Don\IeC {\textquoteright }t Mess Up Your System}{42}{subsection.17.1.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {17.1.2}Preconfigured Images}{42}{subsection.17.1.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {17.1.3}Take It With You}{43}{subsection.17.1.3}}
\@writefile{toc}{\contentsline {section}{\numberline {17.2}Kubernetes Container Deployment}{43}{section.17.2}}
\@writefile{toc}{\contentsline {section}{\numberline {17.3}How to create, start,stop a Container}{44}{section.17.3}}
\@writefile{toc}{\contentsline {section}{\numberline {17.4}Docker micro services?}{44}{section.17.4}}
\@writefile{toc}{\contentsline {section}{\numberline {17.5}Kubernetes}{44}{section.17.5}}
\@writefile{toc}{\contentsline {section}{\numberline {17.6}Why and how to do Docker container orchestration}{44}{section.17.6}}
\@writefile{toc}{\contentsline {chapter}{\numberline {18}REST APIs}{45}{chapter.18}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {18.1}HTTP Post/Get}{45}{section.18.1}}
\@writefile{toc}{\contentsline {section}{\numberline {18.2}API Design}{45}{section.18.2}}
\@writefile{toc}{\contentsline {section}{\numberline {18.3}Implementation}{45}{section.18.3}}
\@writefile{toc}{\contentsline {section}{\numberline {18.4}OAuth security}{45}{section.18.4}}
\@writefile{toc}{\contentsline {chapter}{\numberline {19}Databases}{46}{chapter.19}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {19.1}SQL Databases}{46}{section.19.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.1.1}Database Design}{46}{subsection.19.1.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.1.2}SQL Queries}{46}{subsection.19.1.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.1.3}Stored Procedures}{46}{subsection.19.1.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.1.4}ODBC/JDBC Server Connections}{46}{subsection.19.1.4}}
\@writefile{toc}{\contentsline {section}{\numberline {19.2}NoSQL Stores}{46}{section.19.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.1}KeyValue Stores (HBase)}{46}{subsection.19.2.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.2}Document Store HDFS | available}{46}{subsection.19.2.2}}
\@writefile{lof}{\contentsline {figure}{\numberline {19.1}{\ignorespaces HDFS Master and Data Nodes}}{47}{figure.19.1}}
\newlabel{fig:Bild1}{{19.1}{47}{HDFS Master and Data Nodes}{figure.19.1}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {19.2}{\ignorespaces Distribution of Blocks for a 512MB File}}{48}{figure.19.2}}
\newlabel{fig:Bild1}{{19.2}{48}{Distribution of Blocks for a 512MB File}{figure.19.2}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.3}Document Store MongoDB}{48}{subsection.19.2.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.4}Hive Warehouse}{48}{subsection.19.2.4}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.5}Impala}{48}{subsection.19.2.5}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.6}Kudu}{48}{subsection.19.2.6}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.7}Time Series Databases}{48}{subsection.19.2.7}}
\@writefile{toc}{\contentsline {subsection}{\numberline {19.2.8}MPP Databases (Greenplum)}{48}{subsection.19.2.8}}
\@writefile{toc}{\contentsline {chapter}{\numberline {20}Data Processing / Analytics - Frameworks}{49}{chapter.20}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {20.1}MapReduce}{49}{section.20.1}}
\@writefile{lof}{\contentsline {figure}{\numberline {20.1}{\ignorespaces Mapping of input files and reducing of mapped records}}{50}{figure.20.1}}
\newlabel{fig:Bild1}{{20.1}{50}{Mapping of input files and reducing of mapped records}{figure.20.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.1.1}How does MapReduce work -- available}{51}{subsection.20.1.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.1.2}Example}{51}{subsection.20.1.2}}
\@writefile{lof}{\contentsline {figure}{\numberline {20.2}{\ignorespaces MapReduce Example of Time Series Data}}{52}{figure.20.2}}
\newlabel{fig:Bild1}{{20.2}{52}{MapReduce Example of Time Series Data}{figure.20.2}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.1.3}What is the limitation of MapReduce? -- available}{53}{subsection.20.1.3}}
\@writefile{lof}{\contentsline {figure}{\numberline {20.3}{\ignorespaces The Map Reduce Process}}{53}{figure.20.3}}
\newlabel{fig:Bild1}{{20.3}{53}{The Map Reduce Process}{figure.20.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {20.2}Apache Spark}{53}{section.20.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.1}What is the difference to MapReduce? -- available}{54}{subsection.20.2.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.2}How does Spark fit to Hadoop? -- available}{54}{subsection.20.2.2}}
\@writefile{lof}{\contentsline {figure}{\numberline {20.4}{\ignorespaces Hadoop vs Spark capabilities}}{54}{figure.20.4}}
\newlabel{fig:Bild1}{{20.4}{54}{Hadoop vs Spark capabilities}{figure.20.4}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.3}Where's the difference?}{54}{subsection.20.2.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.4}Spark and Hadoop is a perfect fit}{55}{subsection.20.2.4}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.5}Spark on YARN:}{55}{subsection.20.2.5}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.6}My simple rule of thumb:}{56}{subsection.20.2.6}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.7}Available Languages -- available}{56}{subsection.20.2.7}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.8}How to do stream processing}{56}{subsection.20.2.8}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.9}How to do batch processing}{56}{subsection.20.2.9}}
\@writefile{toc}{\contentsline {subsection}{\numberline {20.2.10}How does Spark use data from Hadoop -- available}{56}{subsection.20.2.10}}
\@writefile{lof}{\contentsline {figure}{\numberline {20.5}{\ignorespaces Spark Using Hadoop Data Locality}}{57}{figure.20.5}}
\newlabel{fig:Bild1}{{20.5}{57}{Spark Using Hadoop Data Locality}{figure.20.5}{}}
\@writefile{toc}{\contentsline {section}{\numberline {20.3}What is a RDD and what is a DataFrame?}{58}{section.20.3}}
\@writefile{toc}{\contentsline {section}{\numberline {20.4}Spark coding with Scala}{58}{section.20.4}}
\@writefile{toc}{\contentsline {section}{\numberline {20.5}Spark coding with Python}{58}{section.20.5}}
\@writefile{toc}{\contentsline {section}{\numberline {20.6}How and why to use SparkSQL?}{58}{section.20.6}}
\@writefile{toc}{\contentsline {section}{\numberline {20.7}Machine Learning on Spark? (Tensor Flow)}{58}{section.20.7}}
\@writefile{toc}{\contentsline {section}{\numberline {20.8}MLlib:}{58}{section.20.8}}
\@writefile{toc}{\contentsline {section}{\numberline {20.9}Spark Setup -- available}{58}{section.20.9}}
\@writefile{toc}{\contentsline {section}{\numberline {20.10}Spark Resource Management -- available}{59}{section.20.10}}
\@writefile{lof}{\contentsline {figure}{\numberline {20.6}{\ignorespaces Spark Resource Management With YARN}}{59}{figure.20.6}}
\newlabel{fig:Bild1}{{20.6}{59}{Spark Resource Management With YARN}{figure.20.6}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {21}Apache Kafka}{60}{chapter.21}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {21.1}Why a message queue tool?}{60}{section.21.1}}
\@writefile{toc}{\contentsline {section}{\numberline {21.2}Kakfa architecture}{60}{section.21.2}}
\@writefile{toc}{\contentsline {section}{\numberline {21.3}What are topics}{60}{section.21.3}}
\@writefile{toc}{\contentsline {section}{\numberline {21.4}What does Zookeeper have to do with Kafka}{60}{section.21.4}}
\@writefile{toc}{\contentsline {section}{\numberline {21.5}How to produce and consume messages}{60}{section.21.5}}
\@writefile{toc}{\contentsline {chapter}{\numberline {22}Machine Learning}{61}{chapter.22}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {22.1}Training and Applying models}{61}{section.22.1}}
\@writefile{toc}{\contentsline {section}{\numberline {22.2}What is deep learning}{61}{section.22.2}}
\@writefile{toc}{\contentsline {section}{\numberline {22.3}How to do Machine Learning in production | available}{61}{section.22.3}}
\@writefile{toc}{\contentsline {section}{\numberline {22.4}Why machine learning in production is harder then you think -- available}{62}{section.22.4}}
\@writefile{toc}{\contentsline {section}{\numberline {22.5}Models Do Not Work Forever}{62}{section.22.5}}
\@writefile{toc}{\contentsline {section}{\numberline {22.6}Where The Platforms That Support This?}{62}{section.22.6}}
\@writefile{toc}{\contentsline {section}{\numberline {22.7}Training Parameter Management}{63}{section.22.7}}
\@writefile{toc}{\contentsline {section}{\numberline {22.8}What\IeC {\textquoteright }s Your Solution?}{63}{section.22.8}}
\@writefile{toc}{\contentsline {section}{\numberline {22.9}How to convince people machine learning works | available}{63}{section.22.9}}
\@writefile{toc}{\contentsline {section}{\numberline {22.10}No Rules, No Physical Models}{64}{section.22.10}}
\@writefile{toc}{\contentsline {section}{\numberline {22.11}You Have The Data. USE IT!}{64}{section.22.11}}
\@writefile{toc}{\contentsline {section}{\numberline {22.12}Data is Stronger Than Opinions}{65}{section.22.12}}
\@writefile{toc}{\contentsline {chapter}{\numberline {23}Data Visualization}{66}{chapter.23}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {23.1}Android \& IOS}{66}{section.23.1}}
\@writefile{toc}{\contentsline {section}{\numberline {23.2}How to design APIs for mobile apps}{66}{section.23.2}}
\@writefile{toc}{\contentsline {section}{\numberline {23.3}How to use Webservers to display content}{66}{section.23.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.3.1}Tomcat}{67}{subsection.23.3.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.3.2}Jetty}{67}{subsection.23.3.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.3.3}NodeRED}{67}{subsection.23.3.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.3.4}React}{67}{subsection.23.3.4}}
\@writefile{toc}{\contentsline {section}{\numberline {23.4}Business Intelligence Tools}{67}{section.23.4}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.4.1}Tableau}{67}{subsection.23.4.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.4.2}PowerBI}{67}{subsection.23.4.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.4.3}Quliksense}{67}{subsection.23.4.3}}
\@writefile{toc}{\contentsline {section}{\numberline {23.5}Identity \& Device Management}{67}{section.23.5}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.5.1}What is a digital twin?}{67}{subsection.23.5.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {23.5.2}Active Directory}{67}{subsection.23.5.2}}
\@writefile{toc}{\contentsline {part}{\numberline {III}Building A Data Platform Example}{68}{part.3}}
\@writefile{toc}{\contentsline {chapter}{\numberline {24}My Big Data Platform Blueprint}{69}{chapter.24}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {paragraph}{\nonumberline THE BLUEPRINT}{69}{section*.4}}
\@writefile{lof}{\contentsline {figure}{\numberline {24.1}{\ignorespaces Platfrom Blueprint}}{69}{figure.24.1}}
\newlabel{fig:Bild1}{{24.1}{69}{Platfrom Blueprint}{figure.24.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {24.1}Ingest}{70}{section.24.1}}
\@writefile{toc}{\contentsline {section}{\numberline {24.2}Analyse / Process}{70}{section.24.2}}
\@writefile{toc}{\contentsline {section}{\numberline {24.3}Store}{71}{section.24.3}}
\@writefile{toc}{\contentsline {section}{\numberline {24.4}Display}{72}{section.24.4}}
\@writefile{toc}{\contentsline {chapter}{\numberline {25}Lambda Architecture}{73}{chapter.25}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {25.1}Batch Processing}{73}{section.25.1}}
\@writefile{lof}{\contentsline {figure}{\numberline {25.1}{\ignorespaces Batch Processing Pipeline}}{73}{figure.25.1}}
\newlabel{fig:Bild1}{{25.1}{73}{Batch Processing Pipeline}{figure.25.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {25.2}Stream Processing}{74}{section.25.2}}
\@writefile{lof}{\contentsline {figure}{\numberline {25.2}{\ignorespaces Stream Processing Pipeline}}{74}{figure.25.2}}
\newlabel{fig:Bild1}{{25.2}{74}{Stream Processing Pipeline}{figure.25.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {25.3}Should you do stream or batch processing?}{74}{section.25.3}}
\@writefile{toc}{\contentsline {section}{\numberline {25.4}Lambda Architecture Alternative}{75}{section.25.4}}
\@writefile{toc}{\contentsline {subsection}{\numberline {25.4.1}Kappa Architecture}{75}{subsection.25.4.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {25.4.2}Kappa Architecture with Kudu}{75}{subsection.25.4.2}}
\@writefile{toc}{\contentsline {chapter}{\numberline {26}Thoughts On Choosing The Target Environment}{76}{chapter.26}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {26.1}Cloud vs On-Premise}{76}{section.26.1}}
\@writefile{toc}{\contentsline {section}{\numberline {26.2}Cloud Native or Independent Vendors}{76}{section.26.2}}
\@writefile{toc}{\contentsline {chapter}{\numberline {27}Thoughts On Choosing A Development Environment}{77}{chapter.27}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {27.1}Cloud As Dev Environment}{77}{section.27.1}}
\@writefile{toc}{\contentsline {section}{\numberline {27.2}Local Dev Environment}{77}{section.27.2}}
\@writefile{toc}{\contentsline {section}{\numberline {27.3}Data Architecture}{77}{section.27.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {27.3.1}Source Data}{77}{subsection.27.3.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {27.3.2}Analytics Requirements For Streaming}{77}{subsection.27.3.2}}
\@writefile{toc}{\contentsline {subsection}{\numberline {27.3.3}Analytics Requirements For Batch Processing}{77}{subsection.27.3.3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {27.3.4}Data Visualization}{77}{subsection.27.3.4}}
\@writefile{toc}{\contentsline {section}{\numberline {27.4}Milestone 1 | Tool Decisions}{77}{section.27.4}}
\@writefile{toc}{\contentsline {part}{\numberline {IV}Case Studies}{78}{part.4}}
\@writefile{toc}{\contentsline {chapter}{\numberline {28}How I do Case Studies}{79}{chapter.28}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{toc}{\contentsline {section}{\numberline {28.1}Data Science @Airbnb}{79}{section.28.1}}
\@writefile{toc}{\contentsline {subsection}{\numberline {28.1.1}Data Science @Amazon}{79}{subsection.28.1.1}}
\@writefile{toc}{\contentsline {section}{\numberline {28.2}Data Science @Baidu}{79}{section.28.2}}
\@writefile{toc}{\contentsline {section}{\numberline {28.3}Data Sciecne @Blackrock}{79}{section.28.3}}
\@writefile{toc}{\contentsline {section}{\numberline {28.4}Data Sciecne @BMW}{80}{section.28.4}}
\@writefile{toc}{\contentsline {section}{\numberline {28.5}Data Sciecne @Booking.com}{80}{section.28.5}}
\@writefile{toc}{\contentsline {section}{\numberline {28.6}Data Science @CERN}{80}{section.28.6}}
\@writefile{toc}{\contentsline {section}{\numberline {28.7}Data Science @Disney}{81}{section.28.7}}
\@writefile{toc}{\contentsline {section}{\numberline {28.8}Data Science @Drivetribe}{81}{section.28.8}}
\@writefile{toc}{\contentsline {section}{\numberline {28.9}Data Science @Dropbox}{81}{section.28.9}}
\@writefile{toc}{\contentsline {section}{\numberline {28.10}Data Science @Ebay}{81}{section.28.10}}
\@writefile{toc}{\contentsline {section}{\numberline {28.11}Data Science @Expedia}{81}{section.28.11}}
\@writefile{toc}{\contentsline {section}{\numberline {28.12}Data Science @Facebook}{81}{section.28.12}}
\@writefile{toc}{\contentsline {subsection}{\numberline {28.12.1}Data Science @Google}{82}{subsection.28.12.1}}
\@writefile{toc}{\contentsline {section}{\numberline {28.13}Data Science @@Grammarly}{82}{section.28.13}}
\@writefile{toc}{\contentsline {section}{\numberline {28.14}Data Science @ING Fraud}{82}{section.28.14}}
\@writefile{toc}{\contentsline {section}{\numberline {28.15}Data Science @Instagram}{82}{section.28.15}}
\@writefile{toc}{\contentsline {section}{\numberline {28.16}Data Science @LinkedIn}{82}{section.28.16}}
\@writefile{toc}{\contentsline {section}{\numberline {28.17}Data Science @Lyft}{82}{section.28.17}}
\@writefile{toc}{\contentsline {section}{\numberline {28.18}Data Science @NASA}{82}{section.28.18}}
\@writefile{toc}{\contentsline {section}{\numberline {28.19}Data Science @Netflix -- available}{83}{section.28.19}}
\@writefile{toc}{\contentsline {paragraph}{\nonumberline The Netflix batch processing pipeline}{83}{section*.5}}
\@writefile{lof}{\contentsline {figure}{\numberline {28.1}{\ignorespaces Old Netflix Batch Processing Pipeline}}{84}{figure.28.1}}
\newlabel{fig:Bild1}{{28.1}{84}{Old Netflix Batch Processing Pipeline}{figure.28.1}{}}
\@writefile{toc}{\contentsline {paragraph}{\nonumberline Know what customers want:}{84}{section*.6}}
\@writefile{toc}{\contentsline {paragraph}{\nonumberline Batch processing is not enough}{84}{section*.7}}
\@writefile{toc}{\contentsline {paragraph}{\nonumberline The trending now feature}{85}{section*.8}}
\@writefile{lof}{\contentsline {figure}{\numberline {28.2}{\ignorespaces Netflix Trending Now Feature}}{85}{figure.28.2}}
\newlabel{fig:Bild1}{{28.2}{85}{Netflix Trending Now Feature}{figure.28.2}{}}
\@writefile{toc}{\contentsline {paragraph}{\nonumberline Netflix real-time streaming architecture}{85}{section*.9}}
\@writefile{lof}{\contentsline {figure}{\numberline {28.3}{\ignorespaces Netflix Streaming Pipeline}}{86}{figure.28.3}}
\newlabel{fig:Bild1}{{28.3}{86}{Netflix Streaming Pipeline}{figure.28.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {28.20}Data Science @OTTO}{86}{section.28.20}}
\@writefile{toc}{\contentsline {section}{\numberline {28.21}Data Science @Paypal}{87}{section.28.21}}
\@writefile{toc}{\contentsline {section}{\numberline {28.22}Data Science @Pinterest}{87}{section.28.22}}
\@writefile{toc}{\contentsline {section}{\numberline {28.23}Data Science @Salesforce}{87}{section.28.23}}
\@writefile{toc}{\contentsline {section}{\numberline {28.24}Data Science @Slack}{87}{section.28.24}}
\@writefile{toc}{\contentsline {section}{\numberline {28.25}Data Science @Spotify}{87}{section.28.25}}
\@writefile{toc}{\contentsline {section}{\numberline {28.26}Data Science @Symantec}{88}{section.28.26}}
\@writefile{toc}{\contentsline {section}{\numberline {28.27}Data Science @Tinder}{88}{section.28.27}}
\@writefile{toc}{\contentsline {section}{\numberline {28.28}Data Science @Twitter}{88}{section.28.28}}
\@writefile{toc}{\contentsline {section}{\numberline {28.29}Data Science @Uber}{88}{section.28.29}}
\@writefile{toc}{\contentsline {section}{\numberline {28.30}Data Science @Upwork}{88}{section.28.30}}
\@writefile{toc}{\contentsline {section}{\numberline {28.31}Data Science @Woot}{88}{section.28.31}}
\@writefile{toc}{\contentsline {section}{\numberline {28.32}Data Science @Zalando}{89}{section.28.32}}
\bibcite{einstein}{1}
\global\csname @altsecnumformattrue\endcsname
\global\@namedef{scr@dte@part@lastmaxnumwidth}{25.4799pt}
\global\@namedef{scr@dte@chapter@lastmaxnumwidth}{18.89992pt}
\global\@namedef{scr@dte@section@lastmaxnumwidth}{31.46349pt}
\global\@namedef{scr@dte@subsection@lastmaxnumwidth}{40.60228pt}
\global\@namedef{scr@dte@figure@lastmaxnumwidth}{25.58855pt}