BUG: patch in weird nested decoding issue, courtesy of @Komnomnomnom

pandas-dev · Jun 11, 2013 · 8e4314d · 8e4314d
1 parent 2697b49
commit 8e4314d
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 4 deletions.
diff --git a/pandas/io/tests/test_json/test_pandas.py b/pandas/io/tests/test_json/test_pandas.py
@@ -314,6 +314,29 @@ def test_date_format(self):
         result = read_json(json,typ='series',parse_dates=True)
         assert_series_equal(result,ts)
 
+    def test_weird_nested_json(self):
+
+        # this used to core dump the parser
+        s = r'''{
+        "status": "success",
+        "data": {
+        "posts": [
+            {
+            "id": 1,
+            "title": "A blog post",
+            "body": "Some useful content"
+            },
+            {
+            "id": 2,
+            "title": "Another blog post",
+            "body": "More content"
+            }
+        ]
+    }
+}'''
+
+        read_json(s)
+
     @network
     @slow
     def test_url(self):

diff --git a/pandas/src/ujson/python/JSONtoObj.c b/pandas/src/ujson/python/JSONtoObj.c
@@ -10,6 +10,7 @@ typedef struct __PyObjectDecoder
     JSONObjectDecoder dec;
 
     void* npyarr;       // Numpy context buffer
+    void* npyarr_addr;  // Ref to npyarr ptr to track DECREF calls
     npy_intp curdim;    // Current array dimension
 
     PyArray_Descr* dtype;
@@ -67,9 +68,7 @@ void Npy_releaseContext(NpyArrContext* npyarr)
         }
         if (npyarr->dec)
         {
-            // Don't set to null, used to make sure we don't Py_DECREF npyarr
-            // in releaseObject
-            // npyarr->dec->npyarr = NULL;
+            npyarr->dec->npyarr = NULL;
             npyarr->dec->curdim = 0;
         }
         Py_XDECREF(npyarr->labels[0]);
@@ -88,6 +87,7 @@ JSOBJ Object_npyNewArray(void* _decoder)
     {
         // start of array - initialise the context buffer
         npyarr = decoder->npyarr = PyObject_Malloc(sizeof(NpyArrContext));
+        decoder->npyarr_addr = npyarr;
 
         if (!npyarr)
         {
@@ -515,7 +515,7 @@ JSOBJ Object_newDouble(double value)
 static void Object_releaseObject(JSOBJ obj, void* _decoder)
 {
     PyObjectDecoder* decoder = (PyObjectDecoder*) _decoder;
-    if (obj != decoder->npyarr)
+    if (obj != decoder->npyarr_addr)
     {
         Py_XDECREF( ((PyObject *)obj));
     }
@@ -555,11 +555,13 @@ PyObject* JSONToObj(PyObject* self, PyObject *args, PyObject *kwargs)
     pyDecoder.dec = dec;
     pyDecoder.curdim = 0;
     pyDecoder.npyarr = NULL;
+    pyDecoder.npyarr_addr = NULL;
 
     decoder = (JSONObjectDecoder*) &pyDecoder;
 
     if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|iiO&", kwlist, &sarg, &numpy, &labelled, PyArray_DescrConverter2, &dtype))
     {
+        Npy_releaseContext(pyDecoder.npyarr);
         return NULL;
     }