forked from clj-python/libpython-clj
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathUsage.html
393 lines (373 loc) · 20.1 KB
/
Usage.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
<!DOCTYPE html PUBLIC ""
"">
<html><head><meta charset="UTF-8" /><title>LibPython-CLJ Usage</title><script async="true" src="https://www.googletagmanager.com/gtag/js?id=G-LN7PG6FJ2D"></script><script>window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-LN7PG6FJ2D');</script><link rel="stylesheet" type="text/css" href="css/default.css" /><link rel="stylesheet" type="text/css" href="highlight/solarized-light.css" /><script type="text/javascript" src="highlight/highlight.min.js"></script><script type="text/javascript" src="js/jquery.min.js"></script><script type="text/javascript" src="js/page_effects.js"></script><script>hljs.initHighlightingOnLoad();</script></head><body><div id="header"><h2>Generated by <a href="https://github.com/weavejester/codox">Codox</a> with <a href="https://github.com/xsc/codox-theme-rdash">RDash UI</a> theme</h2><h1><a href="index.html"><span class="project-title"><span class="project-name">libpython-clj</span> <span class="project-version">2.024</span></span></a></h1></div><div class="sidebar primary"><h3 class="no-link"><span class="inner">Project</span></h3><ul class="index-link"><li class="depth-1 "><a href="index.html"><div class="inner">Index</div></a></li></ul><h3 class="no-link"><span class="inner">Topics</span></h3><ul><li class="depth-1 current"><a href="Usage.html"><div class="inner"><span>LibPython-CLJ Usage</span></div></a></li><li class="depth-1 "><a href="embedded.html"><div class="inner"><span>Embedding Clojure In Python</span></div></a></li><li class="depth-1 "><a href="environments.html"><div class="inner"><span>Python Environments</span></div></a></li><li class="depth-1 "><a href="new-to-clojure.html"><div class="inner"><span>So Many Parenthesis!</span></div></a></li><li class="depth-1 "><a href="scopes-and-gc.html"><div class="inner"><span>Scopes And Garbage Collection</span></div></a></li><li class="depth-1 "><a href="slicing.html"><div class="inner"><span>Slicing And Slices</span></div></a></li></ul><h3 class="no-link"><span class="inner">Namespaces</span></h3><ul><li class="depth-1"><div class="no-link"><div class="inner"><span class="tree"><span class="top"></span><span class="bottom"></span></span><span>libpython-clj2</span></div></div></li><li class="depth-2 branch"><a href="libpython-clj2.codegen.html"><div class="inner"><span class="tree"><span class="top"></span><span class="bottom"></span></span><span>codegen</span></div></a></li><li class="depth-2 branch"><a href="libpython-clj2.embedded.html"><div class="inner"><span class="tree"><span class="top"></span><span class="bottom"></span></span><span>embedded</span></div></a></li><li class="depth-2 branch"><a href="libpython-clj2.java-api.html"><div class="inner"><span class="tree"><span class="top"></span><span class="bottom"></span></span><span>java-api</span></div></a></li><li class="depth-2"><a href="libpython-clj2.python.html"><div class="inner"><span class="tree"><span class="top"></span><span class="bottom"></span></span><span>python</span></div></a></li><li class="depth-3 branch"><a href="libpython-clj2.python.class.html"><div class="inner"><span class="tree"><span class="top"></span><span class="bottom"></span></span><span>class</span></div></a></li><li class="depth-3"><a href="libpython-clj2.python.np-array.html"><div class="inner"><span class="tree"><span class="top"></span><span class="bottom"></span></span><span>np-array</span></div></a></li><li class="depth-2"><a href="libpython-clj2.require.html"><div class="inner"><span class="tree" style="top: -83px;"><span class="top" style="height: 92px;"></span><span class="bottom"></span></span><span>require</span></div></a></li></ul></div><div class="document" id="content"><div class="doc"><div class="markdown"><h1>LibPython-CLJ Usage</h1>
<p>Python objects are essentially two dictionaries, one for 'attributes' and one for
'items'. When you use python and use the '.' operator, you are referencing attributes.
If you use the '[]' operator, then you are referencing items. Attributes are built in,
item access is optional and happens via the <code>__getitem__</code> and <code>__setitem__</code> attributes.
This is important to realize in that the code below doesn't look like python because we are
referencing the item and attribute systems by name and not via '.' or '[]'.</p>
<p>This would result in the following analogous code (full example <a href="#dataframe-access-full-example">further on</a>):</p>
<pre><code class="language-python">table.loc[row_date]
</code></pre>
<pre><code class="language-clojure">(get-item (get-attr table :loc) row-date)
</code></pre>
<h3>Installation</h3>
<h4>Ubuntu</h4>
<pre><code class="language-console">sudo apt install libpython3.6
# numpy and pandas are required for unit tests. Numpy is required for
# zero copy support.
python3.6 -m pip install numpy pandas --user
</code></pre>
<h4>MacOSX</h4>
<p>Python installation instructions <a href="https://docs.python-guide.org/starting/install3/osx/">here</a>.</p>
<h3>Initialize python</h3>
<pre><code class="language-clojure">user> (require '[libpython-clj2.python
:refer [as-python as-jvm
->python ->jvm
get-attr call-attr call-attr-kw
get-item initialize!
run-simple-string
add-module module-dict
import-module
python-type
dir]
:as py])
nil
; Mac and Linux
user> (initialize!)
Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke
INFO: executing python initialize!
Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke
INFO: Library python3.6m found at [:system "python3.6m"]
Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke
INFO: Reference thread starting
:ok
; Windows with Anaconda
(initialize! ; Python executable
:python-executable "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\python.exe"
; Python Library
:library-path "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\python37.dll"
; Anacondas PATH environment to load native dlls of modules (numpy, etc.)
:windows-anaconda-activate-bat "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\Scripts\\activate.bat"
)
...
:ok
</code></pre>
<p>This dynamically finds the python shared library and loads it using output from
the python3 executable on your system. For information about how that works,
please checkout the code
<a href="https://github.com/cnuernber/libpython-clj/blob/master/src/libpython_clj/python/interpreter.clj#L30">here</a>.</p>
<h3>Execute Some Python</h3>
<p><code>*out*</code> and <code>*err*</code> capture python stdout and stderr respectively.</p>
<pre><code class="language-clojure">
user> (run-simple-string "print('hey')")
hey
{:globals
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>},
:locals
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>}}
</code></pre>
<p>The results have been 'bridged' into java meaning they are still python objects but
there are java wrappers over the top of them. For instance, <code>Object.toString</code> forwards
its implementation to the python function <code>__str__</code>.</p>
<pre><code class="language-clojure">(def bridged (run-simple-string "print('hey')"))
(instance? java.util.Map (:globals bridged))
true
user> (:globals bridged)
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>}
</code></pre>
<p>We can get and set global variables here. If we run another string, these are in the
environment. The globals map itself is the global dict of the main module:</p>
<pre><code class="language-clojure">(def main-globals (-> (add-module "__main__")
(module-dict)))
#'user/main-globals
user> main-globals
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>}
user> (keys main-globals)
("__name__"
"__doc__"
"__package__"
"__loader__"
"__spec__"
"__annotations__"
"__builtins__")
user> (get main-globals "__name__")
"__main__"
user> (.put main-globals "my_var" 200)
nil
user> (run-simple-string "print('your variable is:' + str(my_var))")
your variable is:200
{:globals
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200},
:locals
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200}}
</code></pre>
<p>Running Python isn't ever really necessary, however, although it may at times be
convenient. You can call attributes from clojure easily:</p>
<pre><code class="language-clojure">user> (def np (import-module "numpy"))
#'user/np
user> (def ones-ary (call-attr np "ones" [2 3]))
#'user/ones-ary
user> ones-ary
[[1. 1. 1.]
[1. 1. 1.]]
user> (call-attr ones-ary "__len__")
2
user> (vec ones-ary)
[[1. 1. 1.] [1. 1. 1.]]
user> (type (first *1))
:pyobject
user> (get-attr ones-ary "shape")
(2, 3)
user> (vec (get-attr ones-ary "shape"))
[2 3]
user> (dir ones-ary)
["T"
"__abs__"
"__add__"
"__and__"
"__array__"
"__array_finalize__"
...
...
"strides"
"sum"
"swapaxes"
"take"
"tobytes"
"tofile"
"tolist"
"tostring"
"trace"
"transpose"
"var"
"view"]
### DataFrame access full example
Here's how to create Pandas DataFrame and accessing its rows via `loc` in both Python and Clojure:
```python
# Python
import numpy as np
import pandas as pan
dates = pan.date_range("1/1/2000", periods=8)
table = pan.DataFrame(np.random.randn(8, 4), index=dates, columns=["A", "B", "C", "D"])
row_date = pan.date_range(start="2000-01-01", end="2000-01-01")
table.loc[row_date]
</code></pre>
<pre><code class="language-clojure">; Clojure
(require '[libpython-clj2.require :refer [require-python]])
(require-python '[numpy :as np])
(require-python '[pandas :as pan])
(def dates (pan/date_range "1/1/2000" :periods 8))
(def table (pan/DataFrame (call-attr np/random :randn 8 4) :index dates :columns ["A" "B" "C" "D"]))
(def row-date (pan/date_range :start "2000-01-01" :end "2000-01-01"))
(get-item (get-attr table :loc) row-date)
</code></pre>
<h3>Errors</h3>
<p>Errors are caught and an exception is thrown. The error text is saved verbatim
in the exception:</p>
<pre><code class="language-clojure">user> (run-simple-string "print('syntax errrr")
Execution error (ExceptionInfo) at libpython-clj.python.interpreter/check-error-throw (interpreter.clj:260).
File "<string>", line 1
print('syntax errrr
^
SyntaxError: EOL while scanning string literal
</code></pre>
<h3>Some Syntax Sugar</h3>
<pre><code class="language-clojure">user> (py/from-import numpy linspace)
#'user/linspace
user> (linspace 2 3 :num 10)
[2. 2.11111111 2.22222222 2.33333333 2.44444444 2.55555556
2.66666667 2.77777778 2.88888889 3. ]
user> (doc linspace)
-------------------------
user/linspace
Return evenly spaced numbers over a specified interval.
Returns `num` evenly spaced samples, calculated over the
interval [`start`, `stop`].
</code></pre>
<ul>
<li><code>from-import</code> - sugar around python <code>from a import b</code>. Takes multiple b's.</li>
<li><code>import-as</code> - sugar around python <code>import a as b</code>.</li>
<li><code>$a</code> - call an attribute using symbol att name. Keywords map to kwargs</li>
<li><code>$c</code> - call an object mapping keywords to kwargs</li>
</ul>
<h4>Experimental Sugar</h4>
<p>We are trying to find the best way to handle attributes in order to shorten
generic python notebook-type usage. The currently implemented direction is:</p>
<ul>
<li><code>$.</code> - get an attribute. Can pass in symbol, string, or keyword</li>
<li><code>$..</code> - get an attribute. If more args are present, get the attribute on that
result.</li>
</ul>
<pre><code class="language-clojure">user> (py/$. numpy linspace)
<function linspace at 0x7fa6642766a8>
user> (py/$.. numpy random shuffle)
<built-in method shuffle of numpy.random.mtrand.RandomState object at 0x7fa66410cca8>
</code></pre>
<h5>Extra sugar</h5>
<p><code>libpython-clj</code> offers syntactic forms similar to those offered by
Clojure for interacting with Python classes and objects.</p>
<p><strong>Class/object methods</strong>
Where in Clojure you would use <code>(. obj method arg1 arg2 ... argN)</code>,
you can use <code>(py. pyobj method arg1 arg2 ... argN)</code>.</p>
<p>In Python, this is equivalent to <code>pyobj.method(arg1, arg2, ..., argN)</code>.
Concrete examples are shown below.</p>
<p><strong>Class/object attributes</strong>
Where in Clojure you would use <code>(.- obj attr)</code>, you can use
<code>(py.- pyobj attr)</code>.</p>
<p>In Python, this is equivalent to <code>pyobj.attr</code>.
Concrete examples shown below.</p>
<p><strong>Nested attribute access</strong>
To achieve a chain of method/attribute access, use the <code>py..</code> for.</p>
<pre><code class="language-clojure">(py.. (requests/get "http://www.google.com")
-content
(decode "latin-1"))
</code></pre>
<p>(<strong>Note</strong>: requires Python <code>requests</code> module installled)</p>
<p><strong>Examples</strong></p>
<pre><code class="language-clojure">user=> (require '[libpython-clj2.python :as py :refer [py. py.. py.-]])
nil
user=> (require '[libpython-clj2.require :refer [require-python]])
... debug info ...
user=> (require-python '[builtins :as python])
WARNING: AssertionError already refers to: class java.lang.AssertionError in namespace: builtins, being replaced by: #'builtins/AssertionError
WARNING: Exception already refers to: class java.lang.Exception in namespace: builtins, being replaced by: #'builtins/Exception
nil
user=> (def xs (python/list))
#'user/xs
user=> (py. xs append 1)
nil
user=> xs
[1]
user=> (py. xs extend [1 2 3])
nil
user=> xs
[1, 1, 2, 3]
user=> (py. xs __len__)
4
user=> ((py.- xs __len__)) ;; attribute syntax to get then call method
4
user=> (py. xs pop)
3
user=> (py. xs clear)
nil
;; requires Python requests module installed
user=> (require-python 'requests)
nil
user=> (def requests (py/import-module "requests"))
#'user/requests
user=> (py.. requests (get "http://www.google.com") -content (decode "latin-1"))
"<!doctype html><html itemscope=\"\" ... snip ... "
</code></pre>
<h3>Numpy</h3>
<p>Speaking of numpy, you can move data between numpy and java easily.</p>
<pre><code class="language-clojure">(require '[tech.v3.tensor :as dtt])
;;includes the appropriate protocols and multimethod overloads
(require '[libpython-clj2.python.np-array]
;;python objects created now for numpy arrays will be different. So you have to require
;;np-array *before* you create your numpy data.
user> (def ones-ary (py/py. np ones [2 3]))
#'user/ones-ary
user> (def tens-data (dtt/as-tensor ones-ary))
#'user/tens-data
user> tens-data
#tech.v3.tensor<float64>[2 3]
[[1.000 1.000 1.000]
[1.000 1.000 1.000]]
user> (require '[tech.v3.datatype :as dtype])
nil
;;Only constant-time count items can be copied, so vectors and arrays and such.
user> (def ignored (dtype/copy! (vec (repeat 6 5)) tens-data))
#'user/ignored
user> (.put main-globals "ones_ary" ones-ary)
nil
user> (run-simple-string "print(ones_ary)")
[[5. 5. 5.]
[5. 5. 5.]]
{:globals
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200, 'ones_ary': array([[5., 5., 5.],
[5., 5., 5.]])},
:locals
{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200, 'ones_ary': array([[5., 5., 5.],
[5., 5., 5.]])}}
</code></pre>
<p>So heavy data has a zero-copy route. Anything backed by a <code>:native-buffer</code> has a
zero copy pathway to and from numpy. For more information on how this happens,
please refer to the datatype library <a href="https://github.com/techascent/tech.datatype/tree/master/docs">documentation</a>.</p>
<p>Just keep in mind, careless usage of zero copy is going to cause spooky action at a
distance.</p>
<h3>Pickle</h3>
<p>Speaking of numpy, you can pickle python objects and transform the result via numpy and dtype
to a java byte array and back:</p>
<pre><code class="language-clojure">user> (require '[libpython-clj2.python :as py])
nil
user> (py/initialize!)
Sep 03, 2022 11:23:34 AM clojure.tools.logging$eval5948$fn__5951 invoke
INFO: Detecting startup info
Sep 03, 2022 11:23:34 AM clojure.tools.logging$eval5948$fn__5951 invoke
INFO: Startup info {:lib-version "3.9", :java-library-path-addendum "/home/chrisn/miniconda3/lib", :exec-prefix "/home/chrisn/miniconda3", :executable "/home/chrisn/miniconda3/bin/python3", :libnames ("python3.9m" "python3.9"), :prefix "/home/chrisn/miniconda3", :base-prefix "/home/chrisn/miniconda3", :libname "python3.9m", :base-exec-prefix "/home/chrisn/miniconda3", :python-home "/home/chrisn/miniconda3", :version [3 9 1], :platform "linux"}
Sep 03, 2022 11:23:34 AM clojure.tools.logging$eval5948$fn__5951 invoke
INFO: Prefixing java library path: /home/chrisn/miniconda3/lib
Sep 03, 2022 11:23:35 AM clojure.tools.logging$eval5948$fn__5951 invoke
INFO: Loading python library: python3.9
Sep 03, 2022 11:23:35 AM clojure.tools.logging$eval5948$fn__5951 invoke
INFO: Reference thread starting
:ok
user> (def data (py/->python {:a 1 :b 2}))
#'user/data
user> (def pickle (py/import-module "pickle"))
#'user/pickle
user> (def bdata (py/py. pickle dumps data))
#'user/bdata
user> bdata
b'\x80\x04\x95\x11\x00\x00\x00\x00\x00\x00\x00}\x94(\x8c\x01a\x94K\x01\x8c\x01b\x94K\x02u.'
user> (def np (py/import-module "numpy"))
#'user/np
user> (py/py. np frombuffer bdata :dtype "int8")
[-128 4 -107 17 0 0 0 0 0 0 0 125 -108 40
-116 1 97 -108 75 1 -116 1 98 -108 75 2 117 46]
user> (require '[libpython-clj2.python.np-array])
nil
user> (def ary (py/py. np frombuffer bdata :dtype "int8"))
#'user/ary
user> (py/->jvm ary)
#tech.v3.tensor<int8>[28]
[-128 4 -107 17 0 0 0 0 0 0 0 125 -108 40 -116 1 97 -108 75 1 -116 1 98 -108 75 2 117 46]
user> (require '[tech.v3.datatype :as dt])
nil
user> (dt/->byte-array *2)
[-128, 4, -107, 17, 0, 0, 0, 0, 0, 0, 0, 125, -108, 40, -116, 1, 97, -108, 75, 1,
-116, 1, 98, -108, 75, 2, 117, 46]
user> (require '[tech.v3.tensor :as dtt])
nil
user> (dtt/as-tensor *2)
nil
user> (def bdata *3)
#'user/bdata
user> bdata
[-128, 4, -107, 17, 0, 0, 0, 0, 0, 0, 0, 125, -108, 40, -116, 1, 97, -108, 75, 1,
-116, 1, 98, -108, 75, 2, 117, 46]
user> (type bdata)
[B
user> (def tens (dtt/reshape bdata [(dt/ecount bdata)]))
#'user/tens
user> (def pdata (py/->python tens))
#'user/pdata
user> pdata
[-128 4 -107 17 0 0 0 0 0 0 0 125 -108 40
-116 1 97 -108 75 1 -116 1 98 -108 75 2 117 46]
user> (py/python-type *1)
:ndarray
user> (def py-ary *2)
#'user/py-ary
user> (def py-bytes (py/py. py-ary tobytes))
#'user/py-bytes
user> (py/py. pickle loads py-bytes)
{'a': 1, 'b': 2}
user>
</code></pre>
</div></div></div></body></html>