From be294ab42bc51c7588e9ed50982046c03f00f618 Mon Sep 17 00:00:00 2001 From: "Thouis (Ray) Jones" Date: Fri, 25 May 2012 09:59:43 +0200 Subject: [PATCH 1/4] ENH: expose PyDataMem_NEW/FREE/RENEW as numpy API functions with an event hook. Moves PyDataMem_NEW/FREE/RENEW to the external API. Fixes PyDataMem_NEW/RENEW to return void* instead of char*. Replaces PyDataMem_NEW/FREE with NpySortArray_malloc/free in sort.c.src (should be reverted if npysort is moved to be part of multiarraymodule). Adds PyDataMem_SetEventHook which takes a (PyDataMem_EventHookFunc *) as an argument, with signature: void hook(void *old, void *new, size_t size). When not NULL, hook will be called at the end of each PyDataMem_NEW/FREE/RENEW: result = PyDataMem_NEW(size) -> (*hook(NULL, result, size) PyDataMem_FREE(ptr) -> (*hook(ptr, NULL, 0) result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size) Adds tests in multiarray_tests.c.src, driven by tests/test_multiarray.py. --- numpy/core/code_generators/numpy_api.py | 4 ++ numpy/core/include/numpy/ndarraytypes.h | 11 ++-- .../src/multiarray/multiarray_tests.c.src | 57 +++++++++++++++++ numpy/core/src/multiarray/multiarraymodule.c | 64 +++++++++++++++++++ numpy/core/src/npysort/sort.c.src | 18 ++++-- numpy/core/tests/test_multiarray.py | 13 +++- numpy/lib/src/_compiled_base.c | 7 +- 7 files changed, 161 insertions(+), 13 deletions(-) diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index 15b868e23f7a..b4a5ff90c19b 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -346,6 +346,10 @@ 'PyArray_OutputAllowNAConverter': 306, 'PyArray_FailUnlessWriteable': 307, 'PyArray_SetUpdateIfCopyBase': 308, + 'PyDataMem_NEW': 309, + 'PyDataMem_FREE': 310, + 'PyDataMem_RENEW': 311, + 'PyDataMem_SetEventHook': 312, } ufunc_types_api = { diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index d77c5c90f932..62780652a7c4 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -362,10 +362,7 @@ NpyMaskValue_Create(npy_bool exposed, npy_uint8 payload) * allocated. */ - /* Data buffer */ -#define PyDataMem_NEW(size) ((char *)malloc(size)) -#define PyDataMem_FREE(ptr) free(ptr) -#define PyDataMem_RENEW(ptr,size) ((char *)realloc(ptr,size)) + /* Data buffer - PyDataMem_NEW/FREE/RENEW are in multiarraymodule.c */ #define NPY_USE_PYMEM 1 @@ -1976,6 +1973,12 @@ typedef struct { */ } PyArrayInterface; +/* + * This is a function for hooking into the PyDataMem_NEW/FREE/RENEW functions. + * See the documentation for PyDataMem_SetEventHook. + */ +typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size); + #if !(defined(NPY_NO_DEPRECATED_API) && (NPY_API_VERSION <= NPY_NO_DEPRECATED_API)) #include "npy_deprecated_api.h" #endif diff --git a/numpy/core/src/multiarray/multiarray_tests.c.src b/numpy/core/src/multiarray/multiarray_tests.c.src index 15d4c871e60d..09695b28fc12 100644 --- a/numpy/core/src/multiarray/multiarray_tests.c.src +++ b/numpy/core/src/multiarray/multiarray_tests.c.src @@ -375,6 +375,57 @@ clean_ax: return NULL; } +/* PyDataMem_SetHook tests */ +static int malloc_count, free_count; +static PyDataMem_EventHookFunc *old_hook = NULL; + +static void test_hook(void *old, void *new, size_t size) +{ + if (old == NULL) { + malloc_count++; + } + if (size == 0) { + free_count++; + } +} + +static PyObject* +test_pydatamem_seteventhook_start(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + malloc_count = 0; + free_count = 0; + old_hook = PyDataMem_SetEventHook(test_hook); + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject* +test_pydatamem_seteventhook_end(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + static PyDataMem_EventHookFunc *my_hook; + + my_hook = PyDataMem_SetEventHook(old_hook); + if (my_hook != test_hook) { + PyErr_SetString(PyExc_ValueError, + "hook was not the expected test hook"); + return NULL; + } + + if (malloc_count == 0) { + PyErr_SetString(PyExc_ValueError, + "malloc count is zero after test"); + return NULL; + } + if (free_count == 0) { + PyErr_SetString(PyExc_ValueError, + "free count is zero after test"); + return NULL; + } + + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef Multiarray_TestsMethods[] = { {"test_neighborhood_iterator", test_neighborhood_iterator, @@ -382,6 +433,12 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"test_neighborhood_iterator_oob", test_neighborhood_iterator_oob, METH_VARARGS, NULL}, + {"test_pydatamem_seteventhook_start", + test_pydatamem_seteventhook_start, + METH_NOARGS, NULL}, + {"test_pydatamem_seteventhook_end", + test_pydatamem_seteventhook_end, + METH_NOARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 7a657d8b9e23..e9401240ac7c 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3659,6 +3659,70 @@ test_interrupt(PyObject *NPY_UNUSED(self), PyObject *args) return PyInt_FromLong(a); } +/* malloc/free/realloc hook */ +NPY_NO_EXPORT PyDataMem_EventHookFunc *_PyDataMem_eventhook; + +/*NUMPY_API + * Sets the allocation event hook for numpy array data. + * Takes a PyDataMem_EventHookFunc *, which has the signature: + * void hook(void *old, void *new, size_t size). + * Returns a pointer to the previous hook or NULL. + * + * If not NULL, hook will be called at the end of each PyDataMem_NEW/FREE/RENEW: + * result = PyDataMem_NEW(size) -> (*hook)(NULL, result, size) + * PyDataMem_FREE(ptr) -> (*hook)(ptr, NULL, 0) + * result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size) + */ +NPY_NO_EXPORT PyDataMem_EventHookFunc * +PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook) +{ + PyDataMem_EventHookFunc *temp = _PyDataMem_eventhook; + _PyDataMem_eventhook = newhook; + return temp; +} + +/*NUMPY_API + * Allocates memory for array data. + */ +NPY_NO_EXPORT void * +PyDataMem_NEW(size_t size) +{ + void *result; + + result = malloc(size); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(NULL, result, size); + } + return (char *)result; +} + +/*NUMPY_API + * Free memory for array data. + */ +NPY_NO_EXPORT void +PyDataMem_FREE(void *ptr) +{ + free(ptr); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, NULL, 0); + } +} + +/*NUMPY_API + * Reallocate/resize memory for array data. + */ +NPY_NO_EXPORT void * +PyDataMem_RENEW(void *ptr, size_t size) +{ + void *result; + + result = realloc(ptr, size); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, result, size); + } + return (char *)result; +} + static struct PyMethodDef array_module_methods[] = { {"_get_ndarray_c_version", (PyCFunction)array__get_ndarray_c_version, diff --git a/numpy/core/src/npysort/sort.c.src b/numpy/core/src/npysort/sort.c.src index 618259f968c7..5141317485d2 100644 --- a/numpy/core/src/npysort/sort.c.src +++ b/numpy/core/src/npysort/sort.c.src @@ -38,6 +38,12 @@ #define SMALL_MERGESORT 20 #define SMALL_STRING 16 +/* These should be changed to PyDataMem_NEW/FREE if npysort is moved + * to the multiarray directory. + */ +#define NpySortArray_malloc(size) ((char *)malloc(size)) +#define NpySortArray_free(ptr) free(ptr) + /* ***************************************************************************** @@ -335,14 +341,14 @@ mergesort_@suff@(@type@ *start, npy_intp num, void *NOT_USED) pl = start; pr = pl + num; - pw = (@type@ *) PyDataMem_NEW((num/2)*sizeof(@type@)); + pw = (@type@ *) NpySortArray_malloc((num/2)*sizeof(@type@)); if (!pw) { PyErr_NoMemory(); return -1; } mergesort0_@suff@(pl, pr, pw); - PyDataMem_FREE(pw); + NpySortArray_free(pw); return 0; } @@ -476,13 +482,13 @@ mergesort_@suff@(@type@ *start, npy_intp num, PyArrayObject *arr) pl = start; pr = pl + num*len; - pw = (@type@ *) PyDataMem_NEW((num/2)*elsize); + pw = (@type@ *) NpySortArray_malloc((num/2)*elsize); if (!pw) { PyErr_NoMemory(); err = -1; goto fail_0; } - vp = (@type@ *) PyDataMem_NEW(elsize); + vp = (@type@ *) NpySortArray_malloc(elsize); if (!vp) { PyErr_NoMemory(); err = -1; @@ -490,9 +496,9 @@ mergesort_@suff@(@type@ *start, npy_intp num, PyArrayObject *arr) } mergesort0_@suff@(pl, pr, pw, vp, len); - PyDataMem_FREE(vp); + NpySortArray_free(vp); fail_1: - PyDataMem_FREE(pw); + NpySortArray_free(pw); fail_0: return err; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index e9c7a9c4f82a..9b975ea41e44 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -10,7 +10,8 @@ from numpy.compat import asbytes, getexception, strchar from test_print import in_foreign_locale from numpy.core.multiarray_tests import ( - test_neighborhood_iterator, test_neighborhood_iterator_oob + test_neighborhood_iterator, test_neighborhood_iterator_oob, + test_pydatamem_seteventhook_start, test_pydatamem_seteventhook_end, ) from numpy.testing import ( TestCase, run_module_suite, assert_, assert_raises, @@ -2633,6 +2634,16 @@ def test_flat_element_deletion(): except: raise AssertionError +class TestMemEventHook(TestCase): + def test_mem_seteventhook(self): + # The actual tests are within the C code in + # multiarray/multiarray_tests.c.src + test_pydatamem_seteventhook_start() + # force an allocation and free of a numpy array + a = np.zeros(10) + del a + test_pydatamem_seteventhook_end() + if __name__ == "__main__": run_module_suite() diff --git a/numpy/lib/src/_compiled_base.c b/numpy/lib/src/_compiled_base.c index c31ee5cd8bd8..9bb8a613dabf 100644 --- a/numpy/lib/src/_compiled_base.c +++ b/numpy/lib/src/_compiled_base.c @@ -670,7 +670,10 @@ arr_interp(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) /* only pre-calculate slopes if there are relatively few of them. */ if (lenxp <= lenx) { - slopes = (double *) PyDataMem_NEW((lenxp - 1)*sizeof(double)); + slopes = (double *) PyArray_malloc((lenxp - 1)*sizeof(double)); + if (! slopes) { + goto fail; + } NPY_BEGIN_ALLOW_THREADS; for (i = 0; i < lenxp - 1; i++) { slopes[i] = (dy[i + 1] - dy[i])/(dx[i + 1] - dx[i]); @@ -692,7 +695,7 @@ arr_interp(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) } } NPY_END_ALLOW_THREADS; - PyDataMem_FREE(slopes); + PyArray_free(slopes); } else { NPY_BEGIN_ALLOW_THREADS; From 32370937a5692befe4971790cc0b71f72fe08bb1 Mon Sep 17 00:00:00 2001 From: "Thouis (Ray) Jones" Date: Fri, 15 Jun 2012 15:50:15 +0200 Subject: [PATCH 2/4] Add void *user_data to EventHookFunc and SetEventHook --- numpy/core/include/numpy/ndarraytypes.h | 3 +- .../src/multiarray/multiarray_tests.c.src | 28 +++++++++-------- numpy/core/src/multiarray/multiarraymodule.c | 31 +++++++++++++------ 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 62780652a7c4..2a25479b2315 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1977,7 +1977,8 @@ typedef struct { * This is a function for hooking into the PyDataMem_NEW/FREE/RENEW functions. * See the documentation for PyDataMem_SetEventHook. */ -typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size); +typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, + void *user_data); #if !(defined(NPY_NO_DEPRECATED_API) && (NPY_API_VERSION <= NPY_NO_DEPRECATED_API)) #include "npy_deprecated_api.h" diff --git a/numpy/core/src/multiarray/multiarray_tests.c.src b/numpy/core/src/multiarray/multiarray_tests.c.src index 09695b28fc12..a0f70aabc9b7 100644 --- a/numpy/core/src/multiarray/multiarray_tests.c.src +++ b/numpy/core/src/multiarray/multiarray_tests.c.src @@ -376,25 +376,26 @@ clean_ax: } /* PyDataMem_SetHook tests */ -static int malloc_count, free_count; +static int malloc_free_counts[2]; static PyDataMem_EventHookFunc *old_hook = NULL; +static void *old_data; -static void test_hook(void *old, void *new, size_t size) +static void test_hook(void *old, void *new, size_t size, void *user_data) { + int* counters = (int *) user_data; if (old == NULL) { - malloc_count++; + counters[0]++; /* malloc counter */ } if (size == 0) { - free_count++; + counters[1]++; /* free counter */ } } static PyObject* test_pydatamem_seteventhook_start(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) { - malloc_count = 0; - free_count = 0; - old_hook = PyDataMem_SetEventHook(test_hook); + malloc_free_counts[0] = malloc_free_counts[1] = 0; + old_hook = PyDataMem_SetEventHook(test_hook, (void *) malloc_free_counts, &old_data); Py_INCREF(Py_None); return Py_None; } @@ -402,21 +403,22 @@ test_pydatamem_seteventhook_start(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUS static PyObject* test_pydatamem_seteventhook_end(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) { - static PyDataMem_EventHookFunc *my_hook; + PyDataMem_EventHookFunc *my_hook; + void *my_data; - my_hook = PyDataMem_SetEventHook(old_hook); - if (my_hook != test_hook) { + my_hook = PyDataMem_SetEventHook(old_hook, old_data, &my_data); + if ((my_hook != test_hook) || (my_data != (void *) malloc_free_counts)) { PyErr_SetString(PyExc_ValueError, - "hook was not the expected test hook"); + "hook/data was not the expected test hook"); return NULL; } - if (malloc_count == 0) { + if (malloc_free_counts[0] == 0) { PyErr_SetString(PyExc_ValueError, "malloc count is zero after test"); return NULL; } - if (free_count == 0) { + if (malloc_free_counts[1] == 0) { PyErr_SetString(PyExc_ValueError, "free count is zero after test"); return NULL; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index e9401240ac7c..3917c1611d71 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3661,23 +3661,33 @@ test_interrupt(PyObject *NPY_UNUSED(self), PyObject *args) /* malloc/free/realloc hook */ NPY_NO_EXPORT PyDataMem_EventHookFunc *_PyDataMem_eventhook; +NPY_NO_EXPORT void *_PyDataMem_eventhook_user_data; /*NUMPY_API * Sets the allocation event hook for numpy array data. * Takes a PyDataMem_EventHookFunc *, which has the signature: - * void hook(void *old, void *new, size_t size). - * Returns a pointer to the previous hook or NULL. + * void hook(void *old, void *new, size_t size, void *user_data). + * Also takes a void *user_data, and void **old_data. + * + * Returns a pointer to the previous hook or NULL. If old_data is + * non-NULL, the previous user_data pointer will be copied to it. + * * * If not NULL, hook will be called at the end of each PyDataMem_NEW/FREE/RENEW: - * result = PyDataMem_NEW(size) -> (*hook)(NULL, result, size) - * PyDataMem_FREE(ptr) -> (*hook)(ptr, NULL, 0) - * result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size) + * result = PyDataMem_NEW(size) -> (*hook)(NULL, result, size, user_data) + * PyDataMem_FREE(ptr) -> (*hook)(ptr, NULL, 0, user_data) + * result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size, user_data) */ NPY_NO_EXPORT PyDataMem_EventHookFunc * -PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook) +PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook, + void *user_data, void **old_data) { PyDataMem_EventHookFunc *temp = _PyDataMem_eventhook; _PyDataMem_eventhook = newhook; + if (old_data != NULL) { + *old_data = _PyDataMem_eventhook_user_data; + } + _PyDataMem_eventhook_user_data = user_data; return temp; } @@ -3691,7 +3701,8 @@ PyDataMem_NEW(size_t size) result = malloc(size); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(NULL, result, size); + (*_PyDataMem_eventhook)(NULL, result, size, + _PyDataMem_eventhook_user_data); } return (char *)result; } @@ -3704,7 +3715,8 @@ PyDataMem_FREE(void *ptr) { free(ptr); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(ptr, NULL, 0); + (*_PyDataMem_eventhook)(ptr, NULL, 0, + _PyDataMem_eventhook_user_data); } } @@ -3718,7 +3730,8 @@ PyDataMem_RENEW(void *ptr, size_t size) result = realloc(ptr, size); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(ptr, result, size); + (*_PyDataMem_eventhook)(ptr, result, size, + _PyDataMem_eventhook_user_data); } return (char *)result; } From 5b5a0f4999dfac66c9c27160737352c727a3517b Mon Sep 17 00:00:00 2001 From: "Thouis (Ray) Jones" Date: Mon, 18 Jun 2012 11:50:28 +0200 Subject: [PATCH 3/4] Wrap hook functions with GIL, add example. Wraps the SetHook and calls to the hook with the GIL, to prevent races. Adds an example of using the interface for callbacks into python code. --- numpy/core/src/multiarray/multiarraymodule.c | 33 +- tools/allocation_tracking/alloc_hook.pyx | 42 ++ tools/allocation_tracking/setup.py | 9 + tools/allocation_tracking/sorttable.js | 493 ++++++++++++++++++ .../allocation_tracking/track_allocations.py | 131 +++++ 5 files changed, 701 insertions(+), 7 deletions(-) create mode 100644 tools/allocation_tracking/alloc_hook.pyx create mode 100644 tools/allocation_tracking/setup.py create mode 100644 tools/allocation_tracking/sorttable.js create mode 100644 tools/allocation_tracking/track_allocations.py diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 3917c1611d71..54e26c08f1d1 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3672,22 +3672,29 @@ NPY_NO_EXPORT void *_PyDataMem_eventhook_user_data; * Returns a pointer to the previous hook or NULL. If old_data is * non-NULL, the previous user_data pointer will be copied to it. * - * * If not NULL, hook will be called at the end of each PyDataMem_NEW/FREE/RENEW: * result = PyDataMem_NEW(size) -> (*hook)(NULL, result, size, user_data) * PyDataMem_FREE(ptr) -> (*hook)(ptr, NULL, 0, user_data) * result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size, user_data) + * + * When the hook is called, the GIL will be held by the calling + * thread. The hook should be written to be reentrant, if it performs + * operations that might cause new allocation events (such as the + * creation/descruction numpy objects, or creating/destroying Python + * objects which might cause a gc) */ NPY_NO_EXPORT PyDataMem_EventHookFunc * PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook, void *user_data, void **old_data) { + PyGILState_STATE gilstate = PyGILState_Ensure(); PyDataMem_EventHookFunc *temp = _PyDataMem_eventhook; _PyDataMem_eventhook = newhook; if (old_data != NULL) { *old_data = _PyDataMem_eventhook_user_data; } _PyDataMem_eventhook_user_data = user_data; + PyGILState_Release(gilstate); return temp; } @@ -3701,8 +3708,12 @@ PyDataMem_NEW(size_t size) result = malloc(size); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(NULL, result, size, - _PyDataMem_eventhook_user_data); + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(NULL, result, size, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); } return (char *)result; } @@ -3715,8 +3726,12 @@ PyDataMem_FREE(void *ptr) { free(ptr); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(ptr, NULL, 0, - _PyDataMem_eventhook_user_data); + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, NULL, 0, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); } } @@ -3730,8 +3745,12 @@ PyDataMem_RENEW(void *ptr, size_t size) result = realloc(ptr, size); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(ptr, result, size, - _PyDataMem_eventhook_user_data); + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, result, size, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); } return (char *)result; } diff --git a/tools/allocation_tracking/alloc_hook.pyx b/tools/allocation_tracking/alloc_hook.pyx new file mode 100644 index 000000000000..d1e656f90254 --- /dev/null +++ b/tools/allocation_tracking/alloc_hook.pyx @@ -0,0 +1,42 @@ +# A cython wrapper for using python functions as callbacks for +# PyDataMem_SetEventHook. + +cimport numpy as np + +cdef extern from "Python.h": + object PyLong_FromVoidPtr(void *) + void *PyLong_AsVoidPtr(object) + +ctypedef void PyDataMem_EventHookFunc(void *inp, void *outp, size_t size, + void *user_data) +cdef extern from "numpy/arrayobject.h": + PyDataMem_EventHookFunc * \ + PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook, + void *user_data, void **old_data) + +np.import_array() + +cdef void pyhook(void *old, void *new, size_t size, void *user_data): + cdef object pyfunc = user_data + pyfunc(PyLong_FromVoidPtr(old), + PyLong_FromVoidPtr(new), + size) + +class NumpyAllocHook(object): + def __init__(self, callback): + self.callback = callback + + def __enter__(self): + cdef void *old_hook, *old_data + old_hook = \ + PyDataMem_SetEventHook( pyhook, + self.callback, + &old_data) + self.old_hook = PyLong_FromVoidPtr(old_hook) + self.old_data = PyLong_FromVoidPtr(old_data) + + def __exit__(self): + PyDataMem_SetEventHook( \ + PyLong_AsVoidPtr(self.old_hook), + PyLong_AsVoidPtr(self.old_data), + 0) diff --git a/tools/allocation_tracking/setup.py b/tools/allocation_tracking/setup.py new file mode 100644 index 000000000000..4462f9f4ec8c --- /dev/null +++ b/tools/allocation_tracking/setup.py @@ -0,0 +1,9 @@ +from distutils.core import setup +from distutils.extension import Extension +from Cython.Distutils import build_ext +import numpy + +setup( + cmdclass = {'build_ext': build_ext}, + ext_modules = [Extension("alloc_hook", ["alloc_hook.pyx"], + include_dirs=[numpy.get_include()])]) diff --git a/tools/allocation_tracking/sorttable.js b/tools/allocation_tracking/sorttable.js new file mode 100644 index 000000000000..25bccb2b6b91 --- /dev/null +++ b/tools/allocation_tracking/sorttable.js @@ -0,0 +1,493 @@ +/* + SortTable + version 2 + 7th April 2007 + Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/ + + Instructions: + Download this file + Add to your HTML + Add class="sortable" to any table you'd like to make sortable + Click on the headers to sort + + Thanks to many, many people for contributions and suggestions. + Licenced as X11: http://www.kryogenix.org/code/browser/licence.html + This basically means: do what you want with it. +*/ + + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i5' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for fields. + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + if (a[0]==b[0]) return 0; + if (a[0] 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write("\n') + f.write('\n') + f.write("\n") + cols = "event#,lineinfo,bytes allocated,bytes freed,#allocations,#frees,max memory usage,long lived bytes".split(',') + for header in cols: + f.write(" ".format(header)) + f.write("\n\n") + for idx, event in enumerate(self.allocation_trace): + f.write("\n") + event = [idx] + list(event) + for col, val in zip(cols, event): + if col == 'lineinfo': + # special handling + try: + filename, line, module, code, index = val + val = "{0}({1}): {2}".format(filename, line, code[index]) + except: + # sometimes this info is not available (from eval()?) + val = str(val) + f.write(" ".format(val)) + f.write("\n\n") + f.write("
{0}
{0}
\n") + f.close() + + +if __name__ == '__main__': + tracker = AllocationTracker(1000) + with tracker: + for i in range(100): + np.zeros(i * 100) + np.zeros(i * 200) + tracker.write_html("allocations.html") From b42653982c7e8a093369ec9b1cf6088e47c29904 Mon Sep 17 00:00:00 2001 From: "Thouis (Ray) Jones" Date: Fri, 6 Jul 2012 23:43:50 +0200 Subject: [PATCH 4/4] Update C-API version hash --- numpy/core/code_generators/cversions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 99ea072caf20..5bdb33c94756 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -10,4 +10,4 @@ # PyArray_CountNonzero, PyArray_NewLikeArray and PyArray_MatrixProduct2. 0x00000006 = e61d5dc51fa1c6459328266e215d6987 # Version 7 (NumPy 1.7) improved datetime64, misc utilities. -0x00000007 = 1768b6c404a3d5a2a6bfe7c68f89e3aa +0x00000007 = e396ba3912dcf052eaee1b0b203a7724