From 743063ce1303fb94f111b6337e4fac5c1a57b1df Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 6 Nov 2023 17:33:09 +0000 Subject: [PATCH 1/3] add a section for v1 release notes --- docs/src/releasenotes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index ad0ef12f..38710c76 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,5 +1,7 @@ # Release Notes +## Unreleased (v1) + ## Unreleased * `Py` is now treated as a scalar when broadcasting. * Bug fixes. From 63a3e60e7aa3e600256f6d2c9715cbc52a45f6c3 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Tue, 7 Nov 2023 17:35:46 +0000 Subject: [PATCH 2/3] make PythonCall.GC more like Base.GC (#413) Co-authored-by: Christopher Doris --- docs/src/faq.md | 4 ++-- docs/src/releasenotes.md | 1 + src/GC/GC.jl | 40 +++++++++++++++++++++++----------------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/src/faq.md b/docs/src/faq.md index d6b7d117..5c1830a7 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -6,8 +6,8 @@ No. Some rules if you are writing multithreaded code: - Only call Python functions from the first thread. -- You probably also need to call `PythonCall.GC.disable()` on the main thread before any - threaded block of code. Remember to call `PythonCall.GC.enable()` again afterwards. +- You probably also need to call `on=PythonCall.GC.enable(false)` on the main thread before any + threaded block of code. Remember to call `PythonCall.GC.enable(on)` again afterwards. (This is because Julia finalizers can be called from any thread.) - Julia intentionally causes segmentation faults as part of the GC safepoint mechanism. If unhandled, these segfaults will result in termination of the process. To enable signal handling, diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 38710c76..e189f102 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,6 +1,7 @@ # Release Notes ## Unreleased (v1) +* `PythonCall.GC` is now more like `Base.GC`: `enable(true)` replaces `enable()`, `enable(false)` replaces `disable()`, and `gc()` is added. ## Unreleased * `Py` is now treated as a scalar when broadcasting. diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 0d1fa9a8..35c091ae 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -3,7 +3,7 @@ Garbage collection of Python objects. -See `disable` and `enable`. +See [`enable`](@ref) and [`gc`](@ref). """ module GC @@ -13,32 +13,39 @@ const ENABLED = Ref(true) const QUEUE = C.PyPtr[] """ - PythonCall.GC.disable() + PythonCall.GC.enable(on::Bool) -Disable the PythonCall garbage collector. +Control whether garbage collection of Python objects is turned on or off. -This means that whenever a Python object owned by Julia is finalized, it is not immediately -freed but is instead added to a queue of objects to free later when `enable()` is called. +Return the previous GC state. + +Disabling the GC means that whenever a Python object owned by Julia is finalized, it is not +immediately freed but is instead added to a queue of objects to free later when GC is +re-enabled. Like most PythonCall functions, you must only call this from the main thread. """ -function disable() - ENABLED[] = false - return +function enable(on::Bool) + was_on = ENABLED[] + if on + ENABLED[] = true + if !was_on + gc() + end + else + ENABLED[] = false + end + return ans end """ - PythonCall.GC.enable() - -Re-enable the PythonCall garbage collector. + PythonCall.GC.gc() -This frees any Python objects which were finalized while the GC was disabled, and allows -objects finalized in the future to be freed immediately. +Perform garbage collection of Python objects. Like most PythonCall functions, you must only call this from the main thread. """ -function enable() - ENABLED[] = true +function gc() if !isempty(QUEUE) C.with_gil(false) do for ptr in QUEUE @@ -47,9 +54,8 @@ function enable() end end end + empty!(QUEUE) end - empty!(QUEUE) - return end function enqueue(ptr::C.PyPtr) From 8ecb5319b957b531763bb2a47ea4585b07dc847c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Wed, 22 May 2024 17:58:22 +0100 Subject: [PATCH 3/3] partial work on new conversion --- src/Convert/Convert.jl | 15 +- src/Convert/core.jl | 174 +++++++++++++++++++++++ src/ConvertOld/Convert.jl | 29 ++++ src/{Convert => ConvertOld}/ctypes.jl | 0 src/{Convert => ConvertOld}/numpy.jl | 0 src/{Convert => ConvertOld}/pandas.jl | 0 src/{Convert => ConvertOld}/pyconvert.jl | 0 src/{Convert => ConvertOld}/rules.jl | 0 src/PythonCall.jl | 30 ++-- 9 files changed, 219 insertions(+), 29 deletions(-) create mode 100644 src/Convert/core.jl create mode 100644 src/ConvertOld/Convert.jl rename src/{Convert => ConvertOld}/ctypes.jl (100%) rename src/{Convert => ConvertOld}/numpy.jl (100%) rename src/{Convert => ConvertOld}/pandas.jl (100%) rename src/{Convert => ConvertOld}/pyconvert.jl (100%) rename src/{Convert => ConvertOld}/rules.jl (100%) diff --git a/src/Convert/Convert.jl b/src/Convert/Convert.jl index 0cccbf58..87c21eea 100644 --- a/src/Convert/Convert.jl +++ b/src/Convert/Convert.jl @@ -11,19 +11,6 @@ using Dates: Date, Time, DateTime, Second, Millisecond, Microsecond, Nanosecond import ..Core: pyconvert -include("pyconvert.jl") -include("rules.jl") -include("ctypes.jl") -include("numpy.jl") -include("pandas.jl") - -function __init__() - C.with_gil() do - init_pyconvert() - init_ctypes() - init_numpy() - init_pandas() - end -end +include("core.jl") end diff --git a/src/Convert/core.jl b/src/Convert/core.jl new file mode 100644 index 00000000..1ed34e1b --- /dev/null +++ b/src/Convert/core.jl @@ -0,0 +1,174 @@ +# RULES TO IMPLEMENT: +# Jl -> Any +# buffers/arrays -> PyArray / AbstractArray +# Integral -> Integer +# float / Real -> Float64 / AbstractFloat +# complex / Complex -> ComplexF64 / Complex +# range -> StepRange / UnitRange +# tuple -> Tuple +# Mapping -> PyDict / AbstractDict +# Sequece -> PyList / AbstractVector +# Set -> PySet / AbstractSet +# Collection -> PyCollection +# Iterable -> PyIterable ?? +# IOBase -> PyIO +# BaseException -> PyException +# date -> PyDate / Date +# time -> PyTime / Time +# datetime -> PyDateTime / DateTime +# timedelta -> PyTimeDelta / Period / CompoundPeriod +# numpy intXX / uintXX / floatXX -> corresponding type +# ctypes ints / floats / pointers -> corresponding type +# bytes / bytearray -> Vector{UInt8} / Vector{Int8} + +function pyconvert(::Type{T}, x::Py) where {T} + ans = pytryconvert(T, x)::Union{Some{T},Nothing} + if ans === nothing + error("Cannot convert this Python '$(pytype(x).__name__)' to a Julia '$(T)'") + end + something(ans) +end + +function pyconvert(::Type{T}, x::Any) where {T} + return pyconvert(T, Py(x)::Py) +end +export pyconvert + +function pytryconvert(::Type{T}, x::Py) where {T} + t = typeinfo(pytype(x)) + + # subtypes + TNumber = Utils._typeintersect(T, Number) + TAbstractString = Utils._typeintersect(T, AbstractString) + TAbstractChar = Utils._typeintersect(T, AbstractChar) + + + # None -> Nothing / Missing + if (Nothing <: T || Missing <: T) && t.is_none + if Nothing <: T + return Some{T}(nothing) + end + if Missing <: T + return Some{T}(missing) + end + end + + # bool -> Bool + if Bool <: T && t.is_bool + return Some{T}(pybool_asbool(x)) + end + + # Number -> Number + if TNumber != Union{} && t.is_abstract_number + if t.is_abstract_integral + error("not implemented") + elseif t.is_abstract_rational + error("not implemented") + elseif t.is_abstract_real + xfloat = pyfloat_asdouble(x)::Cdouble + ans = tryconvert(T, TNumber, xfloat) + if ans !== nothing + return ans + end + elseif t.is_abstract_complex + xcomplex = pycomplex_ascomplex(x)::Complex{Cdouble} + ans = tryconvert(T, TNumber, xcomplex) + if ans !== nothing + return ans + end + end + end + + # str -> AbstractString / AbstractChar / Symbol + if (TAbstractString != Union{} || TAbstractChar != Union{} || Symbol <: T) && t.is_str + xstr = pystr_asstring(x) + # str -> AbstractString + if TAbstractString != Union{} + ans = tryconvert(T, TAbstractString, xstr) + if ans !== nothing + return ans + end + end + # str -> Symbol + if Symbol <: T + return Some{T}(Symbol(xstr)) + end + # str -> AbstractChar + if TAbstractChar != Union{} && length(xstr) == 1 + xchar = xstr[1]::Char + ans = tryconvert(T, TAbstractChar, xchar) + if ans !== nothing + return ans + end + end + end + + # any -> Py + if Py <: T + return Some{T}(x) + end + nothing +end + +function pytryconvert(::Type{T}, x::Any) where {T} + return pytryconvert(T, Py(x)::Py) +end +export pytryconvert + +Base.@kwdef struct TypeInfo + type::Py + # stdlib concrete types + is_none::Bool = pyissubclass(type, pytype(pybuiltins.None)) + is_bool::Bool = pyissubclass(type, pybuiltins.bool) + is_int::Bool = pyissubclass(type, pybuiltins.int) + is_float::Bool = pyissubclass(type, pybuiltins.float) + is_complex::Bool = pyissubclass(type, pybuiltins.complex) + is_str::Bool = pyissubclass(type, pybuiltins.str) + is_bytes::Bool = pyissubclass(type, pybuiltins.bytes) + is_bytearray::Bool = pyissubclass(type, pybuiltins.bytearray) + is_list::Bool = pyissubclass(type, pybuiltins.list) + is_tuple::Bool = pyissubclass(type, pybuiltins.tuple) + is_dict::Bool = pyissubclass(type, pybuiltins.dict) + is_range::Bool = pyissubclass(type, pybuiltins.range) + is_date::Bool = pyissubclass(type, pyimport("datetime").date) + is_time::Bool = pyissubclass(type, pyimport("datetime").time) + is_datetime::Bool = pyissubclass(type, pyimport("datetime").datetime) + is_timedelta::Bool = pyissubclass(type, pyimport("datetime").timedelta) + is_exception::Bool = pyissubclass(type, pybuiltins.BaseException) + is_io::Bool = pyissubclass(type, pyimport("io").IOBase) + # stdlib abstract types + is_abstract_iterable::Bool = pyissubclass(type, pyimport("collections.abc").Iterable) + is_abstract_collection::Bool = pyissubclass(type, pyimport("collections.abc").Collection) + is_abstract_sequence::Bool = pyissubclass(type, pyimport("collections.abc").Sequence) + is_abstract_set::Bool = pyissubclass(type, pyimport("collections.abc").Set) + is_abstract_mapping::Bool = pyissubclass(type, pyimport("collections.abc").Mapping) + is_abstract_number::Bool = pyissubclass(type, pyimport("numbers").Number) + is_abstract_complex::Bool = pyissubclass(type, pyimport("numbers").Complex) + is_abstract_real::Bool = pyissubclass(type, pyimport("numbers").Real) + is_abstract_rational::Bool = pyissubclass(type, pyimport("numbers").Rational) + is_abstract_integral::Bool = pyissubclass(type, pyimport("numbers").Integral) + # arrays + has_numpy_array_conversion::Bool = pyhasattr(type, "__array__") + has_numpy_array_interface::Bool = pyhasattr(type, "__array_interface__") + has_numpy_array_struct::Bool = pyhasattr(type, "__array_struct__") + has_buffer_protocol::Bool = C.PyType_CheckBuffer(getptr(type)) + is_array_like::Bool = has_numpy_array_conversion || has_numpy_array_interface || has_numpy_array_struct || has_buffer_protocol + # numpy (TODO) + # ctypes (TODO) +end + +const TYPEINFO_CACHE = Dict{Py,TypeInfo}() + +typeinfo(t::Py) = get!(() -> TypeInfo(type=t), TYPEINFO_CACHE, t) + +function tryconvert(::Type{T}, ::Type{S}, x) where {T,S<:T} + try + Some{T}(convert(S, x)::S) + catch + # TODO: only catch some exception types? + nothing + end +end + +tryconvert(::Type{T}, ::Type{T}, x::T) where {T} = Some{T}(x) +tryconvert(::Type{T}, ::Type{S}, x::S) where {T,S<:T} = Some{T}(x) diff --git a/src/ConvertOld/Convert.jl b/src/ConvertOld/Convert.jl new file mode 100644 index 00000000..0cccbf58 --- /dev/null +++ b/src/ConvertOld/Convert.jl @@ -0,0 +1,29 @@ +""" + module PythonCall.Convert + +Implements `pyconvert`. +""" +module Convert + +using ..Core +using ..Core: C, Utils, @autopy, getptr, incref, pynew, PyNULL, pyisnull, pydel!, pyisint, iserrset_ambig, pyisnone, pyisTrue, pyisFalse, pyfloat_asdouble, pycomplex_ascomplex, pyisstr, pystr_asstring, pyisbytes, pybytes_asvector, pybytes_asUTF8string, pyisfloat, pyisrange, pytuple_getitem, unsafe_pynext, pyistuple, pydatetimetype, pytime_isaware, pydatetime_isaware, _base_pydatetime, _base_datetime, errmatches, errclear, errset, pyiscomplex, pythrow, pybool_asbool +using Dates: Date, Time, DateTime, Second, Millisecond, Microsecond, Nanosecond + +import ..Core: pyconvert + +include("pyconvert.jl") +include("rules.jl") +include("ctypes.jl") +include("numpy.jl") +include("pandas.jl") + +function __init__() + C.with_gil() do + init_pyconvert() + init_ctypes() + init_numpy() + init_pandas() + end +end + +end diff --git a/src/Convert/ctypes.jl b/src/ConvertOld/ctypes.jl similarity index 100% rename from src/Convert/ctypes.jl rename to src/ConvertOld/ctypes.jl diff --git a/src/Convert/numpy.jl b/src/ConvertOld/numpy.jl similarity index 100% rename from src/Convert/numpy.jl rename to src/ConvertOld/numpy.jl diff --git a/src/Convert/pandas.jl b/src/ConvertOld/pandas.jl similarity index 100% rename from src/Convert/pandas.jl rename to src/ConvertOld/pandas.jl diff --git a/src/Convert/pyconvert.jl b/src/ConvertOld/pyconvert.jl similarity index 100% rename from src/Convert/pyconvert.jl rename to src/ConvertOld/pyconvert.jl diff --git a/src/Convert/rules.jl b/src/ConvertOld/rules.jl similarity index 100% rename from src/Convert/rules.jl rename to src/ConvertOld/rules.jl diff --git a/src/PythonCall.jl b/src/PythonCall.jl index fdd1774f..a3cb2ce1 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -8,13 +8,13 @@ include("C/C.jl") include("GC/GC.jl") include("Core/Core.jl") include("Convert/Convert.jl") -include("PyMacro/PyMacro.jl") -include("Wrap/Wrap.jl") -include("JlWrap/JlWrap.jl") -include("Compat/Compat.jl") +# include("PyMacro/PyMacro.jl") +# include("Wrap/Wrap.jl") +# include("JlWrap/JlWrap.jl") +# include("Compat/Compat.jl") # re-export everything -for m in [:Core, :Convert, :PyMacro, :Wrap, :JlWrap, :Compat] +for m in [:Core, :Convert] #, :PyMacro, :Wrap, :JlWrap, :Compat] for k in names(@eval($m)) if k != m @eval using .$m: $k @@ -30,16 +30,16 @@ end for k in [:pynew, :pyisnull, :pycopy!, :getptr, :pydel!, :unsafe_pynext, :PyNULL, :CONFIG] @eval using .Core: $k end -for k in [:pyconvert_add_rule, :pyconvert_return, :pyconvert_unconverted, :PYCONVERT_PRIORITY_WRAP, :PYCONVERT_PRIORITY_ARRAY, :PYCONVERT_PRIORITY_CANONICAL, :PYCONVERT_PRIORITY_NORMAL, :PYCONVERT_PRIORITY_FALLBACK] - @eval using .Convert: $k -end -for k in [:event_loop_on, :event_loop_off, :fix_qt_plugin_path] - @eval using .Compat: $k -end +# for k in [:pyconvert_add_rule, :pyconvert_return, :pyconvert_unconverted, :PYCONVERT_PRIORITY_WRAP, :PYCONVERT_PRIORITY_ARRAY, :PYCONVERT_PRIORITY_CANONICAL, :PYCONVERT_PRIORITY_NORMAL, :PYCONVERT_PRIORITY_FALLBACK] +# @eval using .Convert: $k +# end +# for k in [:event_loop_on, :event_loop_off, :fix_qt_plugin_path] +# @eval using .Compat: $k +# end -# not API but used in tests -for k in [:pyjlanytype, :pyjlarraytype, :pyjlvectortype, :pyjlbinaryiotype, :pyjltextiotype, :pyjldicttype, :pyjlmoduletype, :pyjlintegertype, :pyjlrationaltype, :pyjlrealtype, :pyjlcomplextype, :pyjlsettype, :pyjltypetype] - @eval using .JlWrap: $k -end +# # not API but used in tests +# for k in [:pyjlanytype, :pyjlarraytype, :pyjlvectortype, :pyjlbinaryiotype, :pyjltextiotype, :pyjldicttype, :pyjlmoduletype, :pyjlintegertype, :pyjlrationaltype, :pyjlrealtype, :pyjlcomplextype, :pyjlsettype, :pyjltypetype] +# @eval using .JlWrap: $k +# end end