Fixed #24020 -- Refactored SQL compiler to use expressions

Refactored compiler SELECT, GROUP BY and ORDER BY generation.
While there, also refactored select_related() implementation
(get_cached_row() and get_klass_info() are now gone!).

Made get_db_converters() method work on expressions instead of
internal_type. This allows the backend converters to target
specific expressions if need be.

Added query.context, this can be used to set per-query state.

Also changed the signature of database converters. They now accept
context as an argument.

django/contrib/contenttypes/fields.py

@@ -10,7 +10,6 @@
 from django.db.models.base import ModelBase
 from django.db.models.fields.related import ForeignObject, ForeignObjectRel
 from django.db.models.query_utils import PathInfo
-from django.db.models.expressions import Col
 from django.contrib.contenttypes.models import ContentType
 from django.utils.encoding import smart_text, python_2_unicode_compatible
 
@@ -367,7 +366,7 @@ def get_extra_restriction(self, where_class, alias, remote_alias):
         field = self.rel.to._meta.get_field(self.content_type_field_name)
         contenttype_pk = self.get_content_type().pk
         cond = where_class()
-        lookup = field.get_lookup('exact')(Col(remote_alias, field, field), contenttype_pk)
+        lookup = field.get_lookup('exact')(field.get_col(remote_alias), contenttype_pk)
         cond.add(lookup, 'AND')
         return cond
 

django/contrib/gis/db/backends/base.py

@@ -158,10 +158,10 @@ class BaseSpatialOperations(object):
 
     # Default conversion functions for aggregates; will be overridden if implemented
     # for the spatial backend.
-    def convert_extent(self, box):
+    def convert_extent(self, box, srid):
         raise NotImplementedError('Aggregate extent not implemented for this spatial backend.')
 
-    def convert_extent3d(self, box):
+    def convert_extent3d(self, box, srid):
         raise NotImplementedError('Aggregate 3D extent not implemented for this spatial backend.')
 
     def convert_geom(self, geom_val, geom_field):

django/contrib/gis/db/backends/mysql/operations.py

@@ -7,7 +7,6 @@
 
 class MySQLOperations(DatabaseOperations, BaseSpatialOperations):
 
-    compiler_module = 'django.contrib.gis.db.models.sql.compiler'
     mysql = True
     name = 'mysql'
     select = 'AsText(%s)'

django/contrib/gis/db/backends/oracle/operations.py

@@ -52,7 +52,6 @@ def as_sql(self, connection, lookup, template_params, sql_params):
 
 
 class OracleOperations(DatabaseOperations, BaseSpatialOperations):
-    compiler_module = "django.contrib.gis.db.backends.oracle.compiler"
 
     name = 'oracle'
     oracle = True
@@ -111,8 +110,9 @@ class OracleOperations(DatabaseOperations, BaseSpatialOperations):
     def geo_quote_name(self, name):
         return super(OracleOperations, self).geo_quote_name(name).upper()
 
-    def get_db_converters(self, internal_type):
-        converters = super(OracleOperations, self).get_db_converters(internal_type)
+    def get_db_converters(self, expression):
+        converters = super(OracleOperations, self).get_db_converters(expression)
+        internal_type = expression.output_field.get_internal_type()
         geometry_fields = (
             'PointField', 'GeometryField', 'LineStringField',
             'PolygonField', 'MultiPointField', 'MultiLineStringField',
@@ -121,14 +121,23 @@ def get_db_converters(self, internal_type):
         )
         if internal_type in geometry_fields:
             converters.append(self.convert_textfield_value)
+        if hasattr(expression.output_field, 'geom_type'):
+            converters.append(self.convert_geometry)
         return converters
 
-    def convert_extent(self, clob):
+    def convert_geometry(self, value, expression, context):
+        if value:
+            value = Geometry(value)
+            if 'transformed_srid' in context:
+                value.srid = context['transformed_srid']
+        return value
+
+    def convert_extent(self, clob, srid):
         if clob:
             # Generally, Oracle returns a polygon for the extent -- however,
             # it can return a single point if there's only one Point in the
             # table.
-            ext_geom = Geometry(clob.read())
+            ext_geom = Geometry(clob.read(), srid)
             gtype = str(ext_geom.geom_type)
             if gtype == 'Polygon':
                 # Construct the 4-tuple from the coordinates in the polygon.
@@ -226,7 +235,7 @@ def spatial_aggregate_sql(self, agg):
         else:
             sql_template = '%(function)s(SDOAGGRTYPE(%(expressions)s,%(tolerance)s))'
         sql_function = getattr(self, agg_name)
-        return self.select % sql_template, sql_function
+        return sql_template, sql_function
 
     # Routines for getting the OGC-compliant models.
     def geometry_columns(self):

django/contrib/gis/db/backends/postgis/operations.py

@@ -44,7 +44,6 @@ def as_sql(self, connection, lookup, template_params, sql_params):
 
 
 class PostGISOperations(DatabaseOperations, BaseSpatialOperations):
-    compiler_module = 'django.contrib.gis.db.models.sql.compiler'
     name = 'postgis'
     postgis = True
     geography = True
@@ -188,7 +187,7 @@ def check_aggregate_support(self, aggregate):
         agg_name = aggregate.__class__.__name__
         return agg_name in self.valid_aggregates
 
-    def convert_extent(self, box):
+    def convert_extent(self, box, srid):
         """
         Returns a 4-tuple extent for the `Extent` aggregate by converting
         the bounding box text returned by PostGIS (`box` argument), for
@@ -199,7 +198,7 @@ def convert_extent(self, box):
         xmax, ymax = map(float, ur.split())
         return (xmin, ymin, xmax, ymax)
 
-    def convert_extent3d(self, box3d):
+    def convert_extent3d(self, box3d, srid):
         """
         Returns a 6-tuple extent for the `Extent3D` aggregate by converting
         the 3d bounding-box text returned by PostGIS (`box3d` argument), for

django/contrib/gis/db/backends/spatialite/operations.py

@@ -14,7 +14,6 @@
 
 
 class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations):
-    compiler_module = 'django.contrib.gis.db.models.sql.compiler'
     name = 'spatialite'
     spatialite = True
     version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)')
@@ -131,11 +130,11 @@ def check_aggregate_support(self, aggregate):
         agg_name = aggregate.__class__.__name__
         return agg_name in self.valid_aggregates
 
-    def convert_extent(self, box):
+    def convert_extent(self, box, srid):
         """
         Convert the polygon data received from Spatialite to min/max values.
         """
-        shell = Geometry(box).shell
+        shell = Geometry(box, srid).shell
         xmin, ymin = shell[0][:2]
         xmax, ymax = shell[2][:2]
         return (xmin, ymin, xmax, ymax)
@@ -256,7 +255,7 @@ def spatial_aggregate_sql(self, agg):
         agg_name = agg_name.lower()
         if agg_name == 'union':
             agg_name += 'agg'
-        sql_template = self.select % '%(function)s(%(expressions)s)'
+        sql_template = '%(function)s(%(expressions)s)'
         sql_function = getattr(self, agg_name)
         return sql_template, sql_function
 
@@ -268,3 +267,16 @@ def geometry_columns(self):
     def spatial_ref_sys(self):
         from django.contrib.gis.db.backends.spatialite.models import SpatialiteSpatialRefSys
         return SpatialiteSpatialRefSys
+
+    def get_db_converters(self, expression):
+        converters = super(SpatiaLiteOperations, self).get_db_converters(expression)
+        if hasattr(expression.output_field, 'geom_type'):
+            converters.append(self.convert_geometry)
+        return converters
+
+    def convert_geometry(self, value, expression, context):
+        if value:
+            value = Geometry(value)
+            if 'transformed_srid' in context:
+                value.srid = context['transformed_srid']
+        return value

django/contrib/gis/db/models/aggregates.py

@@ -28,7 +28,7 @@ def prepare(self, query=None, allow_joins=True, reuse=None, summarize=False):
             raise ValueError('Geospatial aggregates only allowed on geometry fields.')
         return c
 
-    def convert_value(self, value, connection):
+    def convert_value(self, value, connection, context):
         return connection.ops.convert_geom(value, self.output_field)
 
 
@@ -43,8 +43,8 @@ class Extent(GeoAggregate):
     def __init__(self, expression, **extra):
         super(Extent, self).__init__(expression, output_field=ExtentField(), **extra)
 
-    def convert_value(self, value, connection):
-        return connection.ops.convert_extent(value)
+    def convert_value(self, value, connection, context):
+        return connection.ops.convert_extent(value, context.get('transformed_srid'))
 
 
 class Extent3D(GeoAggregate):
@@ -54,8 +54,8 @@ class Extent3D(GeoAggregate):
     def __init__(self, expression, **extra):
         super(Extent3D, self).__init__(expression, output_field=ExtentField(), **extra)
 
-    def convert_value(self, value, connection):
-        return connection.ops.convert_extent3d(value)
+    def convert_value(self, value, connection, context):
+        return connection.ops.convert_extent3d(value, context.get('transformed_srid'))
 
 
 class MakeLine(GeoAggregate):

django/contrib/gis/db/models/fields.py

@@ -42,7 +42,30 @@ def get_srid_info(srid, connection):
     return _srid_cache[connection.alias][srid]
 
 
-class GeometryField(Field):
+class GeoSelectFormatMixin(object):
+    def select_format(self, compiler, sql, params):
+        """
+        Returns the selection format string, depending on the requirements
+        of the spatial backend.  For example, Oracle and MySQL require custom
+        selection formats in order to retrieve geometries in OGC WKT. For all
+        other fields a simple '%s' format string is returned.
+        """
+        connection = compiler.connection
+        srid = compiler.query.get_context('transformed_srid')
+        if srid:
+            sel_fmt = '%s(%%s, %s)' % (connection.ops.transform, srid)
+        else:
+            sel_fmt = '%s'
+        if connection.ops.select:
+            # This allows operations to be done on fields in the SELECT,
+            # overriding their values -- used by the Oracle and MySQL
+            # spatial backends to get database values as WKT, and by the
+            # `transform` method.
+            sel_fmt = connection.ops.select % sel_fmt
+        return sel_fmt % sql, params
+
+
+class GeometryField(GeoSelectFormatMixin, Field):
     "The base GIS field -- maps to the OpenGIS Specification Geometry type."
 
     # The OpenGIS Geometry name.
@@ -196,7 +219,7 @@ def get_prep_value(self, value):
         else:
             return geom
 
-    def from_db_value(self, value, connection):
+    def from_db_value(self, value, connection, context):
         if value and not isinstance(value, Geometry):
             value = Geometry(value)
         return value
@@ -337,7 +360,7 @@ class GeometryCollectionField(GeometryField):
     description = _("Geometry collection")
 
 
-class ExtentField(Field):
+class ExtentField(GeoSelectFormatMixin, Field):
     "Used as a return value from an extent aggregate"
 
     description = _("Extent Aggregate Field")