f2py bug on OpenBSD "'threads.h': file not found " #19437
Description
Reproducing:
Build NumPy 1.20.3 on OpenBSD using its usual compiler, clang 10.0.1, and (e)gfortran.
The resulting NumPy's f2py is buggy. It fails the test in #18179, and also builds of SciPy using it are
broken, see scipy/scipy#14364
Error message:
here is Python3 and numpy used/built:
$ python3
Python 3.8.8 (default, Apr 19 2021, 10:23:47)
[Clang 10.0.1 ] on openbsd6
Type "help", "copyright", "credits" or "license" for more information.
>>> import sys, numpy; print(numpy.__version__, sys.version)
1.20.3 3.8.8 (default, Apr 19 2021, 10:23:47)
[Clang 10.0.1 ]
which, while running
pytest -sv numpy/f2py/tests/test_callback.py -k 'test_all[t]' -r s -x
give
E /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.c:83:10: fatal error: 'threads.h' file not found
E #include <threads.h>
E ^~~~~~~~~~~
E 1 warning and 1 error generated.
full log:
$ pytest -sv numpy/f2py/tests/test_callback.py -k 'test_all[t]' -r s -x
================================================================================================================================== test session starts ===================================================================================================================================
platform openbsd6 -- Python 3.8.8, pytest-6.2.4, py-1.10.0, pluggy-0.13.1 -- /home/dima/sagetrac-mirror/local/bin/python3
cachedir: .pytest_cache
hypothesis profile 'np.test() profile' -> database=None, deadline=None, print_blob=True, derandomize=True, suppress_health_check=[HealthCheck.data_too_large, HealthCheck.filter_too_much, HealthCheck.too_slow, HealthCheck.return_value, HealthCheck.large_base_example, HealthCheck.not_a_test_method, HealthCheck.function_scoped_fixture]
rootdir: /home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/f2py
plugins: hypothesis-6.14.1
collected 16 items / 14 deselected / 2 selected
numpy/f2py/tests/test_callback.py::TestF77Callback::test_all[t] ERROR
========================================================================================================================================= ERRORS =========================================================================================================================================
_____________________________________________________________________________________________________________________ ERROR at setup of TestF77Callback.test_all[t] ______________________________________________________________________________________________________________________
self = <numpy.f2py.tests.test_callback.TestF77Callback object at 0x7245f658340>
def setup(self):
if sys.platform == 'win32':
pytest.skip('Fails with MinGW64 Gfortran (Issue #9673)')
if self.module is not None:
return
# Check compiler availability first
if not has_c_compiler():
pytest.skip("No C compiler available")
codes = []
if self.sources:
codes.extend(self.sources)
if self.code is not None:
codes.append(self.suffix)
needs_f77 = False
needs_f90 = False
for fn in codes:
if fn.endswith('.f'):
needs_f77 = True
elif fn.endswith('.f90'):
needs_f90 = True
if needs_f77 and not has_f77_compiler():
pytest.skip("No Fortran 77 compiler available")
if needs_f90 and not has_f90_compiler():
pytest.skip("No Fortran 90 compiler available")
# Build the module
if self.code is not None:
> self.module = build_code(self.code, options=self.options,
skip=self.skip, only=self.only,
suffix=self.suffix,
module_name=self.module_name)
numpy/f2py/tests/util.py:351:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
numpy/f2py/tests/util.py:73: in wrapper
memo[key] = func(*a, **kw)
numpy/f2py/tests/util.py:158: in build_code
return build_module([path], options=options, skip=skip, only=only,
numpy/f2py/tests/util.py:73: in wrapper
memo[key] = func(*a, **kw)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
source_files = ['/tmp/tmp1lfeb71d.f'], options = [], skip = [], only = [], module_name = '_test_ext_module_5403'
@_memoize
def build_module(source_files, options=[], skip=[], only=[], module_name=None):
"""
Compile and import a f2py module, built from the given files.
"""
code = ("import sys; sys.path = %s; import numpy.f2py as f2py2e; "
"f2py2e.main()" % repr(sys.path))
d = get_module_dir()
# Copy files
dst_sources = []
f2py_sources = []
for fn in source_files:
if not os.path.isfile(fn):
raise RuntimeError("%s is not a file" % fn)
dst = os.path.join(d, os.path.basename(fn))
shutil.copyfile(fn, dst)
dst_sources.append(dst)
base, ext = os.path.splitext(dst)
if ext in ('.f90', '.f', '.c', '.pyf'):
f2py_sources.append(dst)
# Prepare options
if module_name is None:
module_name = get_temp_module_name()
f2py_opts = ['-c', '-m', module_name] + options + f2py_sources
if skip:
f2py_opts += ['skip:'] + skip
if only:
f2py_opts += ['only:'] + only
# Build
cwd = os.getcwd()
try:
os.chdir(d)
cmd = [sys.executable, '-c', code] + f2py_opts
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = p.communicate()
if p.returncode != 0:
> raise RuntimeError("Running f2py failed: %s\n%s"
% (cmd[4:], asstr(out)))
E RuntimeError: Running f2py failed: ['-m', '_test_ext_module_5403', '/tmp/tmp0yy230c6/tmp1lfeb71d.f']
E running build
E running config_cc
E unifing config_cc, config, build_clib, build_ext, build commands --compiler options
E running config_fc
E unifing config_fc, config, build_clib, build_ext, build commands --fcompiler options
E running build_src
E build_src
E building extension "_test_ext_module_5403" sources
E f2py options: []
E f2py:> /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.c
E creating /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8
E Reading fortran codes...
E Reading file '/tmp/tmp0yy230c6/tmp1lfeb71d.f' (format:fix,strict)
E Line #22 in /tmp/tmp0yy230c6/tmp1lfeb71d.f:" intent(callback) fun"
E analyzeline: missing __user__ module (could be nothing)
E Line #22 in /tmp/tmp0yy230c6/tmp1lfeb71d.f:" intent(callback) fun"
E analyzeline: appending intent(callback) fun to t2 arguments
E Line #52 in /tmp/tmp0yy230c6/tmp1lfeb71d.f:" intent(callback, hide) global_f"
E analyzeline: missing __user__ module (could be nothing)
E Line #52 in /tmp/tmp0yy230c6/tmp1lfeb71d.f:" intent(callback, hide) global_f"
E analyzeline: appending intent(callback) global_f to hidden_callback arguments
E Post-processing...
E Block: _test_ext_module_5403
E Block: t
E Block: fun
E Block: func
E Block: func0
E Block: t2
E Block: fun
E Block: string_callback
E Block: callback
E Block: string_callback_array
E Block: callback
E Block: hidden_callback
E Block: global_f
E Block: hidden_callback2
E Post-processing (stage 2)...
E Building modules...
E Constructing call-back function "cb_fun_in_t__user__routines"
E def fun(): return a
E Constructing call-back function "cb_fun_in_t2__user__routines"
E def fun(): return a
E Constructing call-back function "cb_callback_in_string_callback__user__routines"
E def callback(r): return a
E Constructing call-back function "cb_callback_in_string_callback_array__user__routines"
E def callback(cu,[lencu]): return a
E Constructing call-back function "cb_global_f_in_hidden_callback__user__routines"
E def global_f(a): return r
E Building module "_test_ext_module_5403"...
E Constructing wrapper function "t"...
E a = t(fun,[fun_extra_args])
E Constructing wrapper function "func"...
E a = func(a)
E Constructing wrapper function "func0"...
E a = func0()
E Constructing wrapper function "t2"...
E a = t2(fun,[fun_extra_args])
E Constructing wrapper function "string_callback"...
E a = string_callback(callback,[callback_extra_args])
E Constructing wrapper function "string_callback_array"...
E a = string_callback_array(callback,cu,[lencu,callback_extra_args])
E Constructing wrapper function "hidden_callback"...
E r = hidden_callback(a)
E Constructing wrapper function "hidden_callback2"...
E r = hidden_callback2(a)
E Wrote C/API module "_test_ext_module_5403" to file "/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.c"
E adding '/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/fortranobject.c' to sources.
E adding '/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8' to include_dirs.
E copying /home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/f2py/src/fortranobject.c -> /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8
E copying /home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/f2py/src/fortranobject.h -> /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8
E build_src: building npy-pkg config files
E running build_ext
E customize UnixCCompiler
E customize UnixCCompiler using build_ext
E get_default_fcompiler: matching types: '['gnu', 'gnu95']'
E customize GnuFCompiler
E Found executable /usr/local/bin/egfortran
E gnu: no Fortran 90 compiler found
E Found executable /usr/bin/ld
E Found executable /usr/bin/ar
E Found executable /usr/bin/ranlib
E gnu: no Fortran 90 compiler found
E customize Gnu95FCompiler
E customize Gnu95FCompiler
E customize Gnu95FCompiler using build_ext
E building '_test_ext_module_5403' extension
E compiling C sources
E C compiler: clang -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -O2 -pipe -g -fPIC -O2 -pipe -g -O2 -pipe -g -I/usr/local/include -I/usr/local/include -fPIC
E
E creating /tmp/tmpvdb1a4dd/tmp
E creating /tmp/tmpvdb1a4dd/tmp/tmpvdb1a4dd
E creating /tmp/tmpvdb1a4dd/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8
E compile options: '-DNPY_DISABLE_OPTIMIZATION=1 -I/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8 -I/home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/core/include -I/home/dima/sagetrac-mirror/local/include -I/usr/local/include/python3.8 -c'
E clang: /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.c
E In file included from /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.c:16:
E In file included from /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/fortranobject.h:13:
E In file included from /home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/core/include/numpy/arrayobject.h:4:
E In file included from /home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/core/include/numpy/ndarrayobject.h:12:
E In file included from /home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/core/include/numpy/ndarraytypes.h:1944:
E /home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: "Using deprecated NumPy API, disable it with " "#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" [-W#warnings]
E #warning "Using deprecated NumPy API, disable it with " \
E ^
E /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.c:83:10: fatal error: 'threads.h' file not found
E #include <threads.h>
E ^~~~~~~~~~~
E 1 warning and 1 error generated.
E appenddecl: "intent" not implemented.
E analyzevars: character array "character*8 cu(lencu)" is considered as "character cu(lencu,8)"; "intent(c)" is forced.
E analyzevars: character array "character*8 cu(lencu)" is considered as "character cu(lencu,8)"; "intent(c)" is forced.
E appenddecl: "intent" not implemented.
E appenddecl: "intent" not implemented.
E append_needs: unknown need 'int'
E append_needs: unknown need 'int'
E append_needs: unknown need 'int'
E append_needs: unknown need 'int'
E error: Command "clang -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -O2 -pipe -g -fPIC -O2 -pipe -g -O2 -pipe -g -I/usr/local/include -I/usr/local/include -fPIC -DNPY_DISABLE_OPTIMIZATION=1 -I/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8 -I/home/dima/sagetrac-mirror/local/lib/python3.8/site-packages/numpy/core/include -I/home/dima/sagetrac-mirror/local/include -I/usr/local/include/python3.8 -c /tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.c -o /tmp/tmpvdb1a4dd/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.o -MMD -MF /tmp/tmpvdb1a4dd/tmp/tmpvdb1a4dd/src.openbsd-6.9-amd64-3.8/_test_ext_module_5403module.o.d" failed with exit status 1
numpy/f2py/tests/util.py:133: RuntimeError
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
============================================================================================================================ 14 deselected, 1 error in 6.35s =============================================================================================================================
C compiler info:
__STDC_NO_THREADS__ is undefined and there is no header:
$ cc -v
OpenBSD clang version 10.0.1
Target: amd64-unknown-openbsd6.9
Thread model: posix
InstalledDir: /usr/bin
$ cat t.c
#ifndef __STDC_NO_THREADS__
#include <threads.h>
#endif
int main() {return 1;}
$ cc -c t.c
t.c:2:10: fatal error: 'threads.h' file not found
#include <threads.h>
^~~~~~~~~~~
1 error generated.