gh-117845: Detect libedit hook function signature in configure (#117870)

jmroot · web-flow · commit 8515fd79fef1 · 2024-04-17T11:26:10.000+02:00
Older libedit versions (like Apple's) use a different type signature
for rl_startup_hook and rl_pre_input_hook. Add a configure check to
determine which signature is accepted by introducing the
Py_RL_STARTUP_HOOK_TAKES_ARGS macro in pyconfig.h.
diff --git a/Misc/NEWS.d/next/Build/2024-04-15-08-35-06.gh-issue-117845.IowzyW.rst b/Misc/NEWS.d/next/Build/2024-04-15-08-35-06.gh-issue-117845.IowzyW.rst
@@ -0,0 +1 @@
+Fix building against recent libedit versions by detecting readline hook signatures in :program:`configure`.
diff --git a/Modules/readline.c b/Modules/readline.c
@@ -1041,7 +1041,7 @@ on_hook(PyObject *func)
 }
 
 static int
-#if defined(_RL_FUNCTION_TYPEDEF)
+#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS)
 on_startup_hook(void)
 #else
 on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state))
@@ -1061,7 +1061,7 @@ on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state))
 
 #ifdef HAVE_RL_PRE_INPUT_HOOK
 static int
-#if defined(_RL_FUNCTION_TYPEDEF)
+#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS)
 on_pre_input_hook(void)
 #else
 on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state))
diff --git a/configure b/configure
diff --git a/configure.ac b/configure.ac
@@ -6341,6 +6341,21 @@ AS_VAR_IF([with_readline], [no], [
     # in readline as well as newer editline (April 2023)
     AC_CHECK_TYPES([rl_compdisp_func_t], [], [], [readline_includes])
 
+    # Some editline versions declare rl_startup_hook as taking no args, others
+    # declare it as taking 2.
+    AC_CACHE_CHECK([if rl_startup_hook takes arguments], [ac_cv_readline_rl_startup_hook_takes_args], [
+        AC_COMPILE_IFELSE(
+            [AC_LANG_PROGRAM([readline_includes]
+                [extern int test_hook_func(const char *text, int state);],
+                [rl_startup_hook=test_hook_func;])],
+            [ac_cv_readline_rl_startup_hook_takes_args=yes],
+            [ac_cv_readline_rl_startup_hook_takes_args=no]
+        )
+    ])
+    AS_VAR_IF([ac_cv_readline_rl_startup_hook_takes_args], [yes], [
+      AC_DEFINE([Py_RL_STARTUP_HOOK_TAKES_ARGS], [1], [Define if rl_startup_hook takes arguments])
+    ])
+
     m4_undefine([readline_includes])
   ])dnl WITH_SAVE_ENV()
 ])
diff --git a/pyconfig.h.in b/pyconfig.h.in
@@ -1659,6 +1659,9 @@
    SipHash13: 3, externally defined: 0 */
 #undef Py_HASH_ALGORITHM
 
+/* Define if rl_startup_hook takes arguments */
+#undef Py_RL_STARTUP_HOOK_TAKES_ARGS
+
 /* Define if you want to enable internal statistics gathering. */
 #undef Py_STATS
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Fix building against recent libedit versions by detecting readline hook signatures in :program:`configure`.