Reintroduction of IncompleteJSONError.

Now more robust and with more tests. Fixes isagalaev#29.

ijson/__init__.py

@@ -13,7 +13,7 @@
 also two other backends using the C library yajl in ``ijson.backends`` that have
 the same API and are faster under CPython.
 '''
-from ijson.common import JSONError, ObjectBuilder
+from ijson.common import JSONError, IncompleteJSONError, ObjectBuilder
 import ijson.backends.python as backend
 
 

ijson/backends/__init__.py

@@ -10,8 +10,8 @@ def find_yajl(required):
     version (1, 2, ...).
     '''
     # Importing ``ctypes`` should be in scope of this function to prevent failure
-    # of `backends`` package load in a runtime where ``ctypes`` is not available. 
-    # Example of such environment is Google App Engine (GAE). 
+    # of `backends`` package load in a runtime where ``ctypes`` is not available.
+    # Example of such environment is Google App Engine (GAE).
     from ctypes import util, cdll
 
     so_name = util.find_library('yajl')

ijson/backends/python.py

@@ -47,7 +47,7 @@ def Lexer(f, buf_size=BUFSIZE):
                     except ValueError:
                         data = f.read(buf_size)
                         if not data:
-                            raise common.JSONError('Incomplete string lexeme')
+                            raise common.IncompleteJSONError('Incomplete string lexeme')
                         buf += data
                 yield discarded + pos, buf[pos:end + 1]
                 pos = end + 1
@@ -126,45 +126,51 @@ def parse_value(lexer, symbol=None, pos=0):
             except decimal.InvalidOperation:
                 raise UnexpectedSymbol(symbol, pos)
     except StopIteration:
-        raise common.JSONError('Incomplete JSON data')
+        raise common.IncompleteJSONError('Incomplete JSON data')
 
 
 def parse_array(lexer):
     yield ('start_array', None)
-    pos, symbol = next(lexer)
-    if symbol != ']':
-        while True:
-            for event in parse_value(lexer, symbol, pos):
-                yield event
-            pos, symbol = next(lexer)
-            if symbol == ']':
-                break
-            if symbol != ',':
-                raise UnexpectedSymbol(symbol, pos)
-            pos, symbol = next(lexer)
-    yield ('end_array', None)
+    try:
+        pos, symbol = next(lexer)
+        if symbol != ']':
+            while True:
+                for event in parse_value(lexer, symbol, pos):
+                    yield event
+                pos, symbol = next(lexer)
+                if symbol == ']':
+                    break
+                if symbol != ',':
+                    raise UnexpectedSymbol(symbol, pos)
+                pos, symbol = next(lexer)
+        yield ('end_array', None)
+    except StopIteration:
+        raise common.IncompleteJSONError('Incomplete JSON data')
 
 
 def parse_object(lexer):
     yield ('start_map', None)
-    pos, symbol = next(lexer)
-    if symbol != '}':
-        while True:
-            if symbol[0] != '"':
-                raise UnexpectedSymbol(symbol, pos)
-            yield ('map_key', symbol[1:-1])
-            pos, symbol = next(lexer)
-            if symbol != ':':
-                raise UnexpectedSymbol(symbol, pos)
-            for event in parse_value(lexer, None, pos):
-                yield event
-            pos, symbol = next(lexer)
-            if symbol == '}':
-                break
-            if symbol != ',':
-                raise UnexpectedSymbol(symbol, pos)
-            pos, symbol = next(lexer)
-    yield ('end_map', None)
+    try:
+        pos, symbol = next(lexer)
+        if symbol != '}':
+            while True:
+                if symbol[0] != '"':
+                    raise UnexpectedSymbol(symbol, pos)
+                yield ('map_key', symbol[1:-1])
+                pos, symbol = next(lexer)
+                if symbol != ':':
+                    raise UnexpectedSymbol(symbol, pos)
+                for event in parse_value(lexer, None, pos):
+                    yield event
+                pos, symbol = next(lexer)
+                if symbol == '}':
+                    break
+                if symbol != ',':
+                    raise UnexpectedSymbol(symbol, pos)
+                pos, symbol = next(lexer)
+        yield ('end_map', None)
+    except StopIteration:
+        raise common.IncompleteJSONError('Incomplete JSON data')
 
 
 def basic_parse(file, buf_size=BUFSIZE):

ijson/backends/yajl.py

@@ -90,10 +90,11 @@ def c_callback(context, *args):
                 perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
                 error = cast(perror, c_char_p).value
                 yajl.yajl_free_error(handle, perror)
+                exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
                 raise common.JSONError(error)
             if not buffer and not events:
                 if result == YAJL_INSUFFICIENT_DATA:
-                    raise common.JSONError('YAJL_INSUFFICIENT_DATA')
+                    raise common.IncompleteJSONError('Incomplete JSON data')
                 break
 
             for event in events:

ijson/backends/yajl2.py

@@ -92,7 +92,8 @@ def c_callback(context, *args):
                 perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
                 error = cast(perror, c_char_p).value
                 yajl.yajl_free_error(handle, perror)
-                raise common.JSONError(error.decode('utf-8'))
+                exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
+                raise exception(error.decode('utf-8'))
             if not buffer and not events:
                 break
 

ijson/common.py

@@ -11,6 +11,13 @@ class JSONError(Exception):
     pass
 
 
+class IncompleteJSONError(JSONError):
+    '''
+    Raised when the parser can't read expected data from a stream.
+    '''
+    pass
+
+
 def parse(basic_events):
     '''
     An iterator returning parsing events with the information about their location

tests.py

@@ -83,7 +83,6 @@
     ('end_map', None),
 ]
 SCALAR_JSON = b'0'
-EMPTY_JSON = b''
 INVALID_JSONS = [
     b'["key", "value",]',      # trailing comma
     b'["key"  "value"]',       # no comma
@@ -94,7 +93,14 @@
     b'[1, 2] dangling junk'    # dangling junk
 ]
 YAJL1_PASSING_INVALID = INVALID_JSONS[6]
-INCOMPLETE_JSON = b'"test'
+INCOMPLETE_JSONS = [
+    b'',
+    b'"test',
+    b'[1',
+    b'[1,',
+    b'{"key"',
+    b'{"key":',
+]
 STRINGS_JSON = br'''
 {
     "str1": "",
@@ -135,14 +141,6 @@ def test_int_numbers(self):
         numbers = [value for event, value in events if event == 'number']
         self.assertTrue(all(type(n) is int  for n in numbers))
 
-    def test_empty(self):
-        with self.assertRaises(common.JSONError):
-            list(self.backend.basic_parse(BytesIO(EMPTY_JSON)))
-
-    def test_incomplete(self):
-        with self.assertRaises(common.JSONError):
-            list(self.backend.basic_parse(BytesIO(INCOMPLETE_JSON)))
-
     def test_invalid(self):
         for json in INVALID_JSONS:
             # Yajl1 doesn't complain about additional data after the end
@@ -152,6 +150,11 @@ def test_invalid(self):
             with self.assertRaises(common.JSONError) as cm:
                 list(self.backend.basic_parse(BytesIO(json)))
 
+    def test_incomplete(self):
+        for json in INCOMPLETE_JSONS:
+            with self.assertRaises(common.IncompleteJSONError):
+                list(self.backend.basic_parse(BytesIO(json)))
+
     def test_utf8_split(self):
         buf_size = JSON.index(b'\xd1') + 1
         try: