ARROW-9469: [Python] Make more objects weakrefable

By default, Cython extension classes (defined with "cdef class") don't have a weakref slot, so add one to all of them.

This adds just one memory word to each object, which IMHO is acceptable.

Closes apache#7758 from pitrou/ARROW-9469-py-weakrefable-objects

Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Krisztián Szűcs <[email protected]>

python/pyarrow/_compute.pxd

@@ -22,7 +22,7 @@ from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 
 
-cdef class FunctionOptions:
+cdef class FunctionOptions(_Weakrefable):
 
     cdef const CFunctionOptions* get_options(self) except NULL
 

python/pyarrow/_compute.pyx

@@ -96,7 +96,7 @@ cdef wrap_scalar_aggregate_kernel(const CScalarAggregateKernel* c_kernel):
     return kernel
 
 
-cdef class Kernel:
+cdef class Kernel(_Weakrefable):
 
     def __init__(self):
         raise TypeError("Do not call {}'s constructor directly"
@@ -139,7 +139,7 @@ cdef class ScalarAggregateKernel(Kernel):
                 .format(frombytes(self.kernel.signature.get().ToString())))
 
 
-cdef class Function:
+cdef class Function(_Weakrefable):
     cdef:
         shared_ptr[CFunction] sp_func
         CFunction* base_func
@@ -266,7 +266,7 @@ cdef _pack_compute_args(object values, vector[CDatum]* out):
                             "for compute function".format(type(val)))
 
 
-cdef class FunctionRegistry:
+cdef class FunctionRegistry(_Weakrefable):
     cdef:
         CFunctionRegistry* registry
 
@@ -298,7 +298,7 @@ def call_function(name, args, options=None, memory_pool=None):
     return func.call(args, options=options, memory_pool=memory_pool)
 
 
-cdef class FunctionOptions:
+cdef class FunctionOptions(_Weakrefable):
 
     cdef const CFunctionOptions* get_options(self) except NULL:
         raise NotImplementedError("Unimplemented base options")

python/pyarrow/_csv.pxd

@@ -18,9 +18,10 @@
 # cython: language_level = 3
 
 from pyarrow.includes.libarrow cimport *
+from pyarrow.lib cimport _Weakrefable
 
 
-cdef class ParseOptions:
+cdef class ParseOptions(_Weakrefable):
     cdef:
         CCSVParseOptions options
 

python/pyarrow/_csv.pyx

@@ -43,7 +43,7 @@ cdef unsigned char _single_char(s) except 0:
     return <unsigned char> val
 
 
-cdef class ReadOptions:
+cdef class ReadOptions(_Weakrefable):
     """
     Options for reading CSV files.
 
@@ -159,7 +159,7 @@ cdef class ReadOptions:
         self.options.autogenerate_column_names = value
 
 
-cdef class ParseOptions:
+cdef class ParseOptions(_Weakrefable):
     """
     Options for parsing CSV files.
 
@@ -318,7 +318,7 @@ cdef class ParseOptions:
          self.ignore_empty_lines) = state
 
 
-cdef class _ISO8601:
+cdef class _ISO8601(_Weakrefable):
     """
     A special object indicating ISO-8601 parsing.
     """
@@ -334,7 +334,7 @@ cdef class _ISO8601:
 ISO8601 = _ISO8601()
 
 
-cdef class ConvertOptions:
+cdef class ConvertOptions(_Weakrefable):
     """
     Options for converting CSV data.
 

python/pyarrow/_cuda.pxd

@@ -23,15 +23,15 @@ from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.libarrow_cuda cimport *
 
 
-cdef class Context:
+cdef class Context(_Weakrefable):
     cdef:
         shared_ptr[CCudaContext] context
         int device_number
 
     cdef void init(self, const shared_ptr[CCudaContext]& ctx)
 
 
-cdef class IpcMemHandle:
+cdef class IpcMemHandle(_Weakrefable):
     cdef:
         shared_ptr[CCudaIpcMemHandle] handle
 

python/pyarrow/_cuda.pyx

@@ -24,7 +24,7 @@ from pyarrow.util import get_contiguous_span
 cimport cpython as cp
 
 
-cdef class Context:
+cdef class Context(_Weakrefable):
     """
     CUDA driver context.
     """
@@ -327,7 +327,7 @@ cdef class Context:
                              ' `%s` object' % (type(obj)))
 
 
-cdef class IpcMemHandle:
+cdef class IpcMemHandle(_Weakrefable):
     """A serializable container for a CUDA IPC handle.
     """
     cdef void init(self, shared_ptr[CCudaIpcMemHandle]& h):

python/pyarrow/_dataset.pyx

@@ -77,7 +77,7 @@ cdef CFileSource _make_file_source(object file, FileSystem filesystem=None):
     return c_source
 
 
-cdef class Expression:
+cdef class Expression(_Weakrefable):
 
     cdef:
         shared_ptr[CExpression] wrapped
@@ -234,7 +234,7 @@ _deserialize = Expression._deserialize
 cdef Expression _true = Expression._scalar(True)
 
 
-cdef class Dataset:
+cdef class Dataset(_Weakrefable):
     """
     Collection of data fragments and potentially child datasets.
 
@@ -584,7 +584,7 @@ cdef shared_ptr[CExpression] _insert_implicit_casts(Expression filter,
     )
 
 
-cdef class FileFormat:
+cdef class FileFormat(_Weakrefable):
 
     cdef:
         shared_ptr[CFileFormat] wrapped
@@ -647,7 +647,7 @@ cdef class FileFormat:
             return False
 
 
-cdef class Fragment:
+cdef class Fragment(_Weakrefable):
     """Fragment of data from a Dataset."""
 
     cdef:
@@ -839,7 +839,7 @@ cdef class FileFragment(Fragment):
         return FileFormat.wrap(self.file_fragment.format())
 
 
-cdef class RowGroupInfo:
+cdef class RowGroupInfo(_Weakrefable):
     """A wrapper class for RowGroup information"""
 
     cdef:
@@ -976,7 +976,7 @@ cdef class ParquetFileFragment(FileFragment):
         return [Fragment.wrap(c_fragment) for c_fragment in c_fragments]
 
 
-cdef class ParquetReadOptions:
+cdef class ParquetReadOptions(_Weakrefable):
     """
     Parquet format specific options for reading.
 
@@ -1133,7 +1133,7 @@ cdef class CsvFileFormat(FileFormat):
         return CsvFileFormat, (self.parse_options,)
 
 
-cdef class Partitioning:
+cdef class Partitioning(_Weakrefable):
 
     cdef:
         shared_ptr[CPartitioning] wrapped
@@ -1177,7 +1177,7 @@ cdef class Partitioning:
         return pyarrow_wrap_schema(self.partitioning.schema())
 
 
-cdef class PartitioningFactory:
+cdef class PartitioningFactory(_Weakrefable):
 
     cdef:
         shared_ptr[CPartitioningFactory] wrapped
@@ -1356,7 +1356,7 @@ cdef class HivePartitioning(Partitioning):
             CHivePartitioning.MakeFactory(options))
 
 
-cdef class DatasetFactory:
+cdef class DatasetFactory(_Weakrefable):
     """
     DatasetFactory is used to create a Dataset, inspect the Schema
     of the fragments contained in it, and declare a partitioning.
@@ -1451,7 +1451,7 @@ cdef class DatasetFactory:
         return Dataset.wrap(GetResultValue(result))
 
 
-cdef class FileSystemFactoryOptions:
+cdef class FileSystemFactoryOptions(_Weakrefable):
     """
     Influences the discovery of filesystem paths.
 
@@ -1655,7 +1655,7 @@ cdef class UnionDatasetFactory(DatasetFactory):
         self.union_factory = <CUnionDatasetFactory*> sp.get()
 
 
-cdef class ParquetFactoryOptions:
+cdef class ParquetFactoryOptions(_Weakrefable):
     """
     Influences the discovery of parquet dataset.
 
@@ -1778,7 +1778,7 @@ cdef class ParquetDatasetFactory(DatasetFactory):
         self.parquet_factory = <CParquetDatasetFactory*> sp.get()
 
 
-cdef class ScanTask:
+cdef class ScanTask(_Weakrefable):
     """Read record batches from a range of a single data fragment.
 
     A ScanTask is meant to be a unit of work to be dispatched.
@@ -1851,7 +1851,7 @@ cdef void _populate_builder(const shared_ptr[CScannerBuilder]& ptr,
     check_status(builder.BatchSize(batch_size))
 
 
-cdef class Scanner:
+cdef class Scanner(_Weakrefable):
     """A materialized scan operation with context and options bound.
 
     A scanner is the class that glues the scan tasks, data fragments and data