ENH: hugely improve performance of to_datetime on ISO8601 data pandas…

…-dev#1571
bertrandhaut · Jul 11, 2012 · b061260 · b061260
1 parent 3824af1
commit b061260
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 6 deletions.
diff --git a/RELEASE.rst b/RELEASE.rst
@@ -36,6 +36,8 @@ pandas 0.8.1
 
 **Improvements to existing features**
 
+  - Drastically improve ``to_datetime`` performance on ISO8601 datetime strings
+    (with no time zones) (#1571)
   - Add ability to append hierarchical index levels with ``set_index`` and to
     drop single levels with ``reset_index`` (#1569, #1577)
   - Always apply passed functions in ``resample``, even if upsampling (#1596)

diff --git a/pandas/src/datetime.pyx b/pandas/src/datetime.pyx
@@ -637,7 +637,8 @@ cdef inline _string_to_dts(object val, pandas_datetimestruct* dts):
     if result == -1:
         raise ValueError('Unable to parse %s' % str(val))
 
-def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False):
+def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False,
+                      format=None):
     cdef:
         Py_ssize_t i, n = len(values)
         object val
@@ -668,12 +669,18 @@ def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False):
                 if len(val) == 0:
                     iresult[i] = iNaT
                     continue
+
                 try:
-                    result[i] = parse(val, dayfirst=dayfirst)
-                except Exception:
-                    raise TypeError
-                pandas_datetime_to_datetimestruct(iresult[i], PANDAS_FR_ns,
-                                                  &dts)
+                    _string_to_dts(val, &dts)
+                    iresult[i] = pandas_datetimestruct_to_datetime(PANDAS_FR_ns,
+                                                                   &dts)
+                except ValueError:
+                    try:
+                        result[i] = parse(val, dayfirst=dayfirst)
+                    except Exception:
+                        raise TypeError
+                    pandas_datetime_to_datetimestruct(iresult[i], PANDAS_FR_ns,
+                                                      &dts)
                 _check_dts_bounds(iresult[i], &dts)
         return result
     except TypeError:

diff --git a/vb_suite/timeseries.py b/vb_suite/timeseries.py
@@ -132,3 +132,15 @@ def date_range(start=None, end=None, periods=None, freq=None):
 timeseries_timestamp_downsample_mean = \
     Benchmark("ts.resample('D', how='mean')", setup,
               start_date=datetime(2012, 4, 25))
+
+#----------------------------------------------------------------------
+# to_datetime
+
+setup = common_setup + """
+rng = date_range('1/1/2000', periods=20000, freq='h')
+strings = [x.strftime('%Y-%m-%d %H:%M:%S') for x in rng]
+"""
+
+timeseries_to_datetime_iso8601 = \
+    Benchmark('to_datetime(strings)', setup,
+              start_date=datetime(2012, 7, 11))