diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 99ea072caf20..5bdb33c94756 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -10,4 +10,4 @@ # PyArray_CountNonzero, PyArray_NewLikeArray and PyArray_MatrixProduct2. 0x00000006 = e61d5dc51fa1c6459328266e215d6987 # Version 7 (NumPy 1.7) improved datetime64, misc utilities. -0x00000007 = 1768b6c404a3d5a2a6bfe7c68f89e3aa +0x00000007 = e396ba3912dcf052eaee1b0b203a7724 diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index fd2b9628e90d..c49c3c346fbf 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -324,6 +324,10 @@ 'PyArray_DebugPrint': 285, 'PyArray_FailUnlessWriteable': 286, 'PyArray_SetUpdateIfCopyBase': 287, + 'PyDataMem_NEW': 288, + 'PyDataMem_FREE': 289, + 'PyDataMem_RENEW': 290, + 'PyDataMem_SetEventHook': 291, } ufunc_types_api = { diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 9605c20f8e6d..13483396b7f4 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -327,10 +327,7 @@ struct NpyAuxData_tag { * allocated. */ - /* Data buffer */ -#define PyDataMem_NEW(size) ((char *)malloc(size)) -#define PyDataMem_FREE(ptr) free(ptr) -#define PyDataMem_RENEW(ptr,size) ((char *)realloc(ptr,size)) + /* Data buffer - PyDataMem_NEW/FREE/RENEW are in multiarraymodule.c */ #define NPY_USE_PYMEM 1 @@ -1714,6 +1711,13 @@ typedef struct { */ } PyArrayInterface; +/* + * This is a function for hooking into the PyDataMem_NEW/FREE/RENEW functions. + * See the documentation for PyDataMem_SetEventHook. + */ +typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, + void *user_data); + #if !(defined(NPY_NO_DEPRECATED_API) && (NPY_API_VERSION <= NPY_NO_DEPRECATED_API)) #include "npy_deprecated_api.h" #endif diff --git a/numpy/core/src/multiarray/multiarray_tests.c.src b/numpy/core/src/multiarray/multiarray_tests.c.src index 15d4c871e60d..a0f70aabc9b7 100644 --- a/numpy/core/src/multiarray/multiarray_tests.c.src +++ b/numpy/core/src/multiarray/multiarray_tests.c.src @@ -375,6 +375,59 @@ clean_ax: return NULL; } +/* PyDataMem_SetHook tests */ +static int malloc_free_counts[2]; +static PyDataMem_EventHookFunc *old_hook = NULL; +static void *old_data; + +static void test_hook(void *old, void *new, size_t size, void *user_data) +{ + int* counters = (int *) user_data; + if (old == NULL) { + counters[0]++; /* malloc counter */ + } + if (size == 0) { + counters[1]++; /* free counter */ + } +} + +static PyObject* +test_pydatamem_seteventhook_start(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + malloc_free_counts[0] = malloc_free_counts[1] = 0; + old_hook = PyDataMem_SetEventHook(test_hook, (void *) malloc_free_counts, &old_data); + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject* +test_pydatamem_seteventhook_end(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + PyDataMem_EventHookFunc *my_hook; + void *my_data; + + my_hook = PyDataMem_SetEventHook(old_hook, old_data, &my_data); + if ((my_hook != test_hook) || (my_data != (void *) malloc_free_counts)) { + PyErr_SetString(PyExc_ValueError, + "hook/data was not the expected test hook"); + return NULL; + } + + if (malloc_free_counts[0] == 0) { + PyErr_SetString(PyExc_ValueError, + "malloc count is zero after test"); + return NULL; + } + if (malloc_free_counts[1] == 0) { + PyErr_SetString(PyExc_ValueError, + "free count is zero after test"); + return NULL; + } + + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef Multiarray_TestsMethods[] = { {"test_neighborhood_iterator", test_neighborhood_iterator, @@ -382,6 +435,12 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"test_neighborhood_iterator_oob", test_neighborhood_iterator_oob, METH_VARARGS, NULL}, + {"test_pydatamem_seteventhook_start", + test_pydatamem_seteventhook_start, + METH_NOARGS, NULL}, + {"test_pydatamem_seteventhook_end", + test_pydatamem_seteventhook_end, + METH_NOARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 1ab7823ad597..0180b1f89853 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3430,6 +3430,102 @@ test_interrupt(PyObject *NPY_UNUSED(self), PyObject *args) return PyInt_FromLong(a); } +/* malloc/free/realloc hook */ +NPY_NO_EXPORT PyDataMem_EventHookFunc *_PyDataMem_eventhook; +NPY_NO_EXPORT void *_PyDataMem_eventhook_user_data; + +/*NUMPY_API + * Sets the allocation event hook for numpy array data. + * Takes a PyDataMem_EventHookFunc *, which has the signature: + * void hook(void *old, void *new, size_t size, void *user_data). + * Also takes a void *user_data, and void **old_data. + * + * Returns a pointer to the previous hook or NULL. If old_data is + * non-NULL, the previous user_data pointer will be copied to it. + * + * If not NULL, hook will be called at the end of each PyDataMem_NEW/FREE/RENEW: + * result = PyDataMem_NEW(size) -> (*hook)(NULL, result, size, user_data) + * PyDataMem_FREE(ptr) -> (*hook)(ptr, NULL, 0, user_data) + * result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size, user_data) + * + * When the hook is called, the GIL will be held by the calling + * thread. The hook should be written to be reentrant, if it performs + * operations that might cause new allocation events (such as the + * creation/descruction numpy objects, or creating/destroying Python + * objects which might cause a gc) + */ +NPY_NO_EXPORT PyDataMem_EventHookFunc * +PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook, + void *user_data, void **old_data) +{ + PyGILState_STATE gilstate = PyGILState_Ensure(); + PyDataMem_EventHookFunc *temp = _PyDataMem_eventhook; + _PyDataMem_eventhook = newhook; + if (old_data != NULL) { + *old_data = _PyDataMem_eventhook_user_data; + } + _PyDataMem_eventhook_user_data = user_data; + PyGILState_Release(gilstate); + return temp; +} + +/*NUMPY_API + * Allocates memory for array data. + */ +NPY_NO_EXPORT void * +PyDataMem_NEW(size_t size) +{ + void *result; + + result = malloc(size); + if (_PyDataMem_eventhook != NULL) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(NULL, result, size, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); + } + return (char *)result; +} + +/*NUMPY_API + * Free memory for array data. + */ +NPY_NO_EXPORT void +PyDataMem_FREE(void *ptr) +{ + free(ptr); + if (_PyDataMem_eventhook != NULL) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, NULL, 0, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); + } +} + +/*NUMPY_API + * Reallocate/resize memory for array data. + */ +NPY_NO_EXPORT void * +PyDataMem_RENEW(void *ptr, size_t size) +{ + void *result; + + result = realloc(ptr, size); + if (_PyDataMem_eventhook != NULL) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, result, size, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); + } + return (char *)result; +} + static struct PyMethodDef array_module_methods[] = { {"_get_ndarray_c_version", (PyCFunction)array__get_ndarray_c_version, diff --git a/numpy/core/src/npysort/sort.c.src b/numpy/core/src/npysort/sort.c.src index 618259f968c7..5141317485d2 100644 --- a/numpy/core/src/npysort/sort.c.src +++ b/numpy/core/src/npysort/sort.c.src @@ -38,6 +38,12 @@ #define SMALL_MERGESORT 20 #define SMALL_STRING 16 +/* These should be changed to PyDataMem_NEW/FREE if npysort is moved + * to the multiarray directory. + */ +#define NpySortArray_malloc(size) ((char *)malloc(size)) +#define NpySortArray_free(ptr) free(ptr) + /* ***************************************************************************** @@ -335,14 +341,14 @@ mergesort_@suff@(@type@ *start, npy_intp num, void *NOT_USED) pl = start; pr = pl + num; - pw = (@type@ *) PyDataMem_NEW((num/2)*sizeof(@type@)); + pw = (@type@ *) NpySortArray_malloc((num/2)*sizeof(@type@)); if (!pw) { PyErr_NoMemory(); return -1; } mergesort0_@suff@(pl, pr, pw); - PyDataMem_FREE(pw); + NpySortArray_free(pw); return 0; } @@ -476,13 +482,13 @@ mergesort_@suff@(@type@ *start, npy_intp num, PyArrayObject *arr) pl = start; pr = pl + num*len; - pw = (@type@ *) PyDataMem_NEW((num/2)*elsize); + pw = (@type@ *) NpySortArray_malloc((num/2)*elsize); if (!pw) { PyErr_NoMemory(); err = -1; goto fail_0; } - vp = (@type@ *) PyDataMem_NEW(elsize); + vp = (@type@ *) NpySortArray_malloc(elsize); if (!vp) { PyErr_NoMemory(); err = -1; @@ -490,9 +496,9 @@ mergesort_@suff@(@type@ *start, npy_intp num, PyArrayObject *arr) } mergesort0_@suff@(pl, pr, pw, vp, len); - PyDataMem_FREE(vp); + NpySortArray_free(vp); fail_1: - PyDataMem_FREE(pw); + NpySortArray_free(pw); fail_0: return err; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 82ead8958c5c..73abc81b9731 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -10,7 +10,8 @@ from numpy.compat import asbytes, getexception, strchar from test_print import in_foreign_locale from numpy.core.multiarray_tests import ( - test_neighborhood_iterator, test_neighborhood_iterator_oob + test_neighborhood_iterator, test_neighborhood_iterator_oob, + test_pydatamem_seteventhook_start, test_pydatamem_seteventhook_end, ) from numpy.testing import ( TestCase, run_module_suite, assert_, assert_raises, @@ -2649,6 +2650,16 @@ def test_flat_element_deletion(): except: raise AssertionError +class TestMemEventHook(TestCase): + def test_mem_seteventhook(self): + # The actual tests are within the C code in + # multiarray/multiarray_tests.c.src + test_pydatamem_seteventhook_start() + # force an allocation and free of a numpy array + a = np.zeros(10) + del a + test_pydatamem_seteventhook_end() + if __name__ == "__main__": run_module_suite() diff --git a/numpy/lib/src/_compiled_base.c b/numpy/lib/src/_compiled_base.c index c31ee5cd8bd8..9bb8a613dabf 100644 --- a/numpy/lib/src/_compiled_base.c +++ b/numpy/lib/src/_compiled_base.c @@ -670,7 +670,10 @@ arr_interp(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) /* only pre-calculate slopes if there are relatively few of them. */ if (lenxp <= lenx) { - slopes = (double *) PyDataMem_NEW((lenxp - 1)*sizeof(double)); + slopes = (double *) PyArray_malloc((lenxp - 1)*sizeof(double)); + if (! slopes) { + goto fail; + } NPY_BEGIN_ALLOW_THREADS; for (i = 0; i < lenxp - 1; i++) { slopes[i] = (dy[i + 1] - dy[i])/(dx[i + 1] - dx[i]); @@ -692,7 +695,7 @@ arr_interp(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) } } NPY_END_ALLOW_THREADS; - PyDataMem_FREE(slopes); + PyArray_free(slopes); } else { NPY_BEGIN_ALLOW_THREADS; diff --git a/tools/allocation_tracking/alloc_hook.pyx b/tools/allocation_tracking/alloc_hook.pyx new file mode 100644 index 000000000000..d1e656f90254 --- /dev/null +++ b/tools/allocation_tracking/alloc_hook.pyx @@ -0,0 +1,42 @@ +# A cython wrapper for using python functions as callbacks for +# PyDataMem_SetEventHook. + +cimport numpy as np + +cdef extern from "Python.h": + object PyLong_FromVoidPtr(void *) + void *PyLong_AsVoidPtr(object) + +ctypedef void PyDataMem_EventHookFunc(void *inp, void *outp, size_t size, + void *user_data) +cdef extern from "numpy/arrayobject.h": + PyDataMem_EventHookFunc * \ + PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook, + void *user_data, void **old_data) + +np.import_array() + +cdef void pyhook(void *old, void *new, size_t size, void *user_data): + cdef object pyfunc = user_data + pyfunc(PyLong_FromVoidPtr(old), + PyLong_FromVoidPtr(new), + size) + +class NumpyAllocHook(object): + def __init__(self, callback): + self.callback = callback + + def __enter__(self): + cdef void *old_hook, *old_data + old_hook = \ + PyDataMem_SetEventHook( pyhook, + self.callback, + &old_data) + self.old_hook = PyLong_FromVoidPtr(old_hook) + self.old_data = PyLong_FromVoidPtr(old_data) + + def __exit__(self): + PyDataMem_SetEventHook( \ + PyLong_AsVoidPtr(self.old_hook), + PyLong_AsVoidPtr(self.old_data), + 0) diff --git a/tools/allocation_tracking/setup.py b/tools/allocation_tracking/setup.py new file mode 100644 index 000000000000..4462f9f4ec8c --- /dev/null +++ b/tools/allocation_tracking/setup.py @@ -0,0 +1,9 @@ +from distutils.core import setup +from distutils.extension import Extension +from Cython.Distutils import build_ext +import numpy + +setup( + cmdclass = {'build_ext': build_ext}, + ext_modules = [Extension("alloc_hook", ["alloc_hook.pyx"], + include_dirs=[numpy.get_include()])]) diff --git a/tools/allocation_tracking/sorttable.js b/tools/allocation_tracking/sorttable.js new file mode 100644 index 000000000000..25bccb2b6b91 --- /dev/null +++ b/tools/allocation_tracking/sorttable.js @@ -0,0 +1,493 @@ +/* + SortTable + version 2 + 7th April 2007 + Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/ + + Instructions: + Download this file + Add to your HTML + Add class="sortable" to any table you'd like to make sortable + Click on the headers to sort + + Thanks to many, many people for contributions and suggestions. + Licenced as X11: http://www.kryogenix.org/code/browser/licence.html + This basically means: do what you want with it. +*/ + + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i5' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for fields. + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + if (a[0]==b[0]) return 0; + if (a[0] 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write("\n') + f.write('\n') + f.write("\n") + cols = "event#,lineinfo,bytes allocated,bytes freed,#allocations,#frees,max memory usage,long lived bytes".split(',') + for header in cols: + f.write(" ".format(header)) + f.write("\n\n") + for idx, event in enumerate(self.allocation_trace): + f.write("\n") + event = [idx] + list(event) + for col, val in zip(cols, event): + if col == 'lineinfo': + # special handling + try: + filename, line, module, code, index = val + val = "{0}({1}): {2}".format(filename, line, code[index]) + except: + # sometimes this info is not available (from eval()?) + val = str(val) + f.write(" ".format(val)) + f.write("\n\n") + f.write("
{0}
{0}
\n") + f.close() + + +if __name__ == '__main__': + tracker = AllocationTracker(1000) + with tracker: + for i in range(100): + np.zeros(i * 100) + np.zeros(i * 200) + tracker.write_html("allocations.html")