Workaround dumping problem on PaX-enabled Linux kernels

* configure.in (PAXCTL): Check for paxctl.

* src/Makefile.in (PAXCTL): Define.
(temacs$(EXEEXT)): Disable memory randomization for the temacs
binary via PaX flags if the paxctl utility is available.
(emacs$(EXEEXT), bootstrap-emacs$(EXEEXT)):
Restore PaX flags to their default.

Fixes: debbugs:11398

4 files changed, 24 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index 95d9ecbf01b..4aeffbcc4e9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2012-06-03  Ulrich Müller  <ulm@gentoo.org>
+
+	* configure.in (PAXCTL): Check for paxctl. (Bug#11398)
+
 2012-06-01  Paul Eggert  <eggert@cs.ucla.edu>
 
 	Remove --disable-maintainer-mode option from 'configure'.  (Bug#11555)
diff --git a/configure.in b/configure.in
index 055db30772e..07925142bc1 100644
--- a/configure.in
+++ b/configure.in
@@ -699,6 +699,8 @@ AC_PATH_PROG(INSTALL_INFO, install-info, :,
   $PATH$PATH_SEPARATOR/usr/sbin$PATH_SEPARATOR/sbin)
 dnl Don't use GZIP, which is used by gzip for additional parameters.
 AC_PATH_PROG(GZIP_PROG, gzip)
+AC_PATH_PROG(PAXCTL, paxctl,,
+  [$PATH$PATH_SEPARATOR/sbin$PATH_SEPARATOR/usr/sbin])
 
 
 ## Need makeinfo >= 4.7 (?) to build the manuals.
diff --git a/src/ChangeLog b/src/ChangeLog
index 4eaf2d49f36..7f8a356eb71 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,11 @@
+2012-06-03  Ulrich Müller  <ulm@gentoo.org>
+
+	* Makefile.in (PAXCTL): Define.
+	(temacs$(EXEEXT)): Disable memory randomization for the temacs
+	binary via PaX flags if the paxctl utility is available.
+	(emacs$(EXEEXT), bootstrap-emacs$(EXEEXT)):
+	Restore PaX flags to their default.  (Bug#11398)
+
 2012-06-03  Chong Yidong  <cyd@gnu.org>
 
 	* xdisp.c (decode_mode_spec_coding): Display a space for a unibyte
diff --git a/src/Makefile.in b/src/Makefile.in
index 27199578fc0..47c7d5cfbc2 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -113,6 +113,12 @@ TEMACS_LDFLAGS = $(LD_SWITCH_SYSTEM) $(LD_SWITCH_SYSTEM_TEMACS)
 ## $LDFLAGS or empty if NS_IMPL_GNUSTEP (for some reason).
 TEMACS_LDFLAGS2 = @TEMACS_LDFLAGS2@
 
+## If available, the full path to the paxctl program.
+## On grsecurity/PaX systems, unexec will fail due to a gap between
+## the bss section and the heap.  This can be prevented by disabling
+## memory randomization in temacs with "paxctl -r".  See bug#11398.
+PAXCTL = @PAXCTL@
+
 ## Some systems define this to request special libraries.
 LIBS_SYSTEM=@LIBS_SYSTEM@
 
@@ -403,6 +409,7 @@ emacs$(EXEEXT): temacs$(EXEEXT) $(etc)/DOC $(lisp) $(leimdir)/leim-list.el
 	    -f list-load-path-shadows || true; \
 	else \
 	  LC_ALL=C $(RUN_TEMACS) -batch -l loadup dump || exit 1; \
+	  test "X$(PAXCTL)" = X || $(PAXCTL) -zex emacs$(EXEEXT); \
 	  ln -f emacs$(EXEEXT) bootstrap-emacs$(EXEEXT); \
 	  ./emacs -batch -f list-load-path-shadows || true; \
 	fi
@@ -450,6 +457,8 @@ $(lib)/libgnu.a: $(config_h)
 temacs$(EXEEXT): $(START_FILES) stamp-oldxmenu $(obj) $(otherobj) $(lib)/libgnu.a
 	$(CC) $(LD_FIRSTFLAG) $(ALL_CFLAGS) $(TEMACS_LDFLAGS) $(TEMACS_LDFLAGS2) \
 	  -o temacs $(START_FILES) $(obj) $(otherobj) $(lib)/libgnu.a $(LIBES)
+	test "$(CANNOT_DUMP)" = "yes" || \
+	  test "X$(PAXCTL)" = X || $(PAXCTL) -r temacs$(EXEEXT)
 
 ## The following oldxmenu-related rules are only (possibly) used if
 ## HAVE_X11 && !USE_GTK, but there is no harm in always defining them
@@ -597,6 +606,7 @@ bootstrap-emacs$(EXEEXT): temacs$(EXEEXT)
 	  ln -f temacs$(EXEEXT) bootstrap-emacs$(EXEEXT); \
 	else \
 	  $(RUN_TEMACS) --batch --load loadup bootstrap || exit 1; \
+	  test "X$(PAXCTL)" = X || $(PAXCTL) -zex emacs$(EXEEXT); \
 	  mv -f emacs$(EXEEXT) bootstrap-emacs$(EXEEXT); \
 	fi
 	@: Compile some files earlier to speed up further compilation.