openjdk: Add support for grsecurity

pkgs/development/compilers/openjdk/default.nix

@@ -1,5 +1,5 @@
 { stdenv, fetchurl, unzip, zip, procps, coreutils, alsaLib, ant, freetype, cups
-, which, jdk, nettools, xorg
+, which, jdk, nettools, xorg, file
 , fontconfig, cpio, cacert, perl, setJavaClassPath }:
 
 let
@@ -19,6 +19,9 @@ let
 
   build = "43";
 
+  # On x86 for heap sizes over 700MB disable SEGMEXEC and PAGEEXEC as well.
+  paxflags = if stdenv.isi686 then "msp" else "m";
+
 in
 
 stdenv.mkDerivation rec {
@@ -35,7 +38,7 @@ stdenv.mkDerivation rec {
     [ unzip procps ant which zip cpio nettools alsaLib
       xorg.libX11 xorg.libXt xorg.libXext xorg.libXrender xorg.libXtst
       xorg.libXi xorg.libXinerama xorg.libXcursor xorg.lndir
-      fontconfig perl
+      fontconfig perl file
     ];
 
   NIX_LDFLAGS = "-lfontconfig -lXcursor -lXinerama";
@@ -49,7 +52,7 @@ stdenv.mkDerivation rec {
       openjdk/{jdk,corba}/make/common/shared/Defs-utils.gmk
   '';
 
-  patches = [ ./cppflags-include-fix.patch ./fix-java-home.patch ];
+  patches = [ ./cppflags-include-fix.patch ./fix-java-home.patch ./paxctl.patch ];
 
   NIX_NO_SELF_RPATH = true;
 
@@ -72,6 +75,14 @@ stdenv.mkDerivation rec {
 
   configurePhase = "true";
 
+  preBuild = ''
+    # We also need to PaX-mark in the middle of the build
+    substituteInPlace hotspot/make/linux/makefiles/launcher.make \
+       --replace XXX_PAXFLAGS_XXX ${paxflags}
+    substituteInPlace jdk/make/common/Program.gmk  \
+       --replace XXX_PAXFLAGS_XXX ${paxflags}
+  '';
+
   installPhase = ''
     mkdir -p $out/lib/openjdk $out/share $jre/lib/openjdk
 
@@ -98,6 +109,14 @@ stdenv.mkDerivation rec {
     rm -rf $out/lib/openjdk/jre/bin
     ln -s $out/lib/openjdk/bin $out/lib/openjdk/jre/bin
 
+    # Set PaX markings
+    exes=$(file $out/lib/openjdk/bin/* $jre/lib/openjdk/jre/bin/* 2> /dev/null | grep -E 'ELF.*(executable|shared object)' | sed -e 's/: .*$//')
+    echo "to mark: *$exes*"
+    for file in $exes; do
+      echo "marking *$file*"
+      paxmark ${paxflags} "$file"
+    done
+
     # Remove duplicate binaries.
     for i in $(cd $out/lib/openjdk/bin && echo *); do
       if [ "$i" = java ]; then continue; fi

pkgs/development/compilers/openjdk/paxctl.patch

@@ -0,0 +1,28 @@
+diff --git a/hotspot/make/linux/makefiles/launcher.make b/hotspot/make/linux/makefiles/launcher.make
+index 34bbcd6..41b9332 100644
+--- a/hotspot/make/linux/makefiles/launcher.make
++++ b/hotspot/make/linux/makefiles/launcher.make
+@@ -83,6 +83,8 @@ $(LAUNCHER): $(OBJS) $(LIBJVM) $(LAUNCHER_MAPFILE)
+ 	$(QUIETLY) echo Linking launcher...
+ 	$(QUIETLY) $(LINK_LAUNCHER/PRE_HOOK)
+ 	$(QUIETLY) $(LINK_LAUNCHER) $(LFLAGS_LAUNCHER) -o $@ $(OBJS) $(LIBS_LAUNCHER)
++	paxctl -c $(LAUNCHER)
++	paxctl -zex -XXX_PAXFLAGS_XXX $(LAUNCHER)
+ 	$(QUIETLY) $(LINK_LAUNCHER/POST_HOOK)
+ 
+ $(LAUNCHER): $(LAUNCHER_SCRIPT)
+diff --git a/jdk/make/common/Program.gmk b/jdk/make/common/Program.gmk
+index 091800d..1de8cb4 100644
+--- a/jdk/make/common/Program.gmk
++++ b/jdk/make/common/Program.gmk
+@@ -60,6 +60,10 @@ ACTUAL_PROGRAM      = $(ACTUAL_PROGRAM_DIR)/$(ACTUAL_PROGRAM_NAME)
+ program_default_rule: all
+ 
+ program: $(ACTUAL_PROGRAM)
++	if [[ "$(PROGRAM)" = "java" ]]; then \
++		paxctl -c $(ACTUAL_PROGRAM); \
++		paxctl -zex -XXX_PAXFLAGS_XXX $(ACTUAL_PROGRAM); \
++	fi
+ 
+ # Work-around for missing processor specific mapfiles
+ ifndef CROSS_COMPILE_ARCH