SOLARIS: prevent BAD TRAP panic with Studio 12.5

Starting with Solaris Studio 12.3, it is documented that Solaris kernel
modules (such as libafs) must not use any floating point, vector, or
SIMD/SSE instructions on x86 hardware.  However, each new Studio
compiler release (12.4 and especially 12.5) is more likely to use these
types of instructions by default.

If the libafs kernel module includes any forbidden kernel instructions,
Solaris will panic the system with:
  BAD TRAP:  type=7 (#nm Device not available)

Provide a new autoconfig test to specify the required compiler options
(-xvector=%none -xregs=no%float) when building the OpenAFS kernel module
for Solaris, so that no invalid x86 instructions are used.

In addition, reinstate default kernel module optimization for Solaris.
It had been disabled in commit 80592c53cb
to address this same issue in Studio 12.3 and 12.4.  However, Studio
12.5 started using some SSE instructions even with no optimization.

This commit has been tested with OpenAFS master and Studio 12.5 at all
optimization levels (none, -xO1 through -xO5) and verified to contain no
XMM register instructions via the following command:
  $ gobjdump -dlr libafs64.o | grep xmm | wc -l

Change-Id: Ic3c7860f7d524162fd9178a1dab5dd223722ee43
Reviewed-on: https://gerrit.openafs.org/12558
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>

acinclude.m4

@@ -462,6 +462,8 @@ case $system in
 		MKAFS_OSTYPE=SOLARIS
                 AC_MSG_RESULT(sun4)
 		SOLARIS_PATH_CC
+		SOLARIS_CC_TAKES_XVECTOR_NONE
+		AC_SUBST(SOLARIS_CC_KOPTS)
 		SOLARIS_UFSVFS_HAS_DQRWLOCK
 		SOLARIS_FS_HAS_FS_ROLLED
 		SOLARIS_SOLOOKUP_TAKES_SOCKPARAMS

src/cf/osconf.m4

@@ -540,6 +540,7 @@ case $AFS_SYSNAME in
 		CFLAGS="$CFLAGS ${XARCHFLAGS}"
 		LD="/usr/ccs/bin/ld"
 		MT_CFLAGS='-mt'
+		KERN_OPTMZ="-xO3"
 		PAM_CFLAGS="-KPIC"
 		PAM_LIBS="-lc -lpam -lsocket -lnsl -lm"
 		SHLIB_CFLAGS="-KPIC"
@@ -567,10 +568,12 @@ if test "x$enable_optimize_kernel" = "x" ; then
   AS_CASE([$AFS_SYSNAME],
     [sunx86_510|sunx86_511],
       dnl Somewhere around Solaris Studio 12.*, the compiler started adding SSE
-      dnl instructions to optimized code, without any ability to turn it off.
-      dnl So just default to not optimizing kernel code for the relevant
-      dnl platforms, until we get a better autoconf test for this.
-      [enable_optimize_kernel=no],
+      dnl instructions to optimized code, without any known way to turn it off.
+      dnl To cope, this condition was added to change the default to
+      dnl 'no'.
+      dnl Now that we have an autoconf test to allow disabling the SSE
+      dnl optimizations, it's safe to once more default to 'yes' here.
+      [enable_optimize_kernel=yes],
     [enable_optimize_kernel=yes])
 fi
 

src/cf/solaris-test1.m4

@@ -0,0 +1,38 @@
+dnl This test is for the Solaris x86 kernel module
+dnl build.  They prevent newer Solaris compilers (12.3
+dnl and up) from using XMM registers (SIMD instructions)
+dnl and floating point registers, which are invalid in
+dnl Solaris kernel code.
+dnl Without this, Solaris may panic in libafs with:
+dnl   BAD TRAP: type=7 (#nm Device not available)
+dnl
+dnl
+AC_DEFUN([SOLARIS_CC_TAKES_XVECTOR_NONE], [
+  AC_CACHE_CHECK([if $CC accepts -xvector=%none],
+    [ac_cv_solaris_cc_takes_xvector_none],
+    [save_CFLAGS="$CFLAGS"
+     CFLAGS="$CFLAGS -xvector=%none"
+     AC_TRY_COMPILE([],
+		    [int x;],
+		    [ac_cv_solaris_cc_takes_xvector_none=yes],
+		    [ac_cv_solaris_cc_takes_xvector_none=no])
+     CFLAGS="$save_CFLAGS"
+    ])
+
+dnl -xvector=%none first appeared in Studio 11, but has only been
+dnl documented as required for Solaris x86 kernel code since Studio
+dnl 12.3.  Studio 12.3 is when the compiler started making more
+dnl aggressive optimizations by using SSE/SIMD instructions with XMM
+dnl (floating point/ SIMD) registers.  Although -xvector=%none is
+dnl required to prevent these optimizations, it is not sufficient.
+dnl Experiments have shown that -xregs=no%float is also needed to
+dnl 1) eliminate a few optimizations not squelched by -xvector=%none,
+dnl and 2) prevent actual use of floating point types in the kernel
+dnl module.  -xregs=no%float has been present since before Studio 8, so
+dnl it is safe to assume its presence when -xvector=%none is present.
+dnl
+
+  AS_IF([test "$ac_cv_solaris_cc_takes_xvector_none" = "yes"],
+        [SOLARIS_CC_KOPTS="-xvector=%none -xregs=no%float "])
+])
+

src/crypto/hcrypto/kernel/config.h

@@ -97,3 +97,8 @@ static_inline int close(int d) {return -1;}
 #endif
 static_inline int gettimeofday(struct timeval *tp, void *tzp)
     {if (tp == NULL) return -1; tp->tv_sec = osi_Time(); tp->tv_usec = 0; return 0;}
+
+#ifdef AFS_SUN5_ENV
+/* workaround to allow --disable-optimize-kernel on Solaris */
+#define double int
+#endif

src/libafs/MakefileProto.SOLARIS.in

@@ -51,7 +51,7 @@ KDEFS_32 =
 KDEFS_64 = -xarch=amd64 -xmodel=kernel
 
 <all>
-CFLAGS=-I. -I.. -I${TOP_OBJDIR}/src/config ${FSINCLUDES} $(DEFINES) $(KDEFS) $(KOPTS) ${DBUG}
+CFLAGS=-I. -I.. -I${TOP_OBJDIR}/src/config ${FSINCLUDES} $(DEFINES) $(KDEFS) @SOLARIS_CC_KOPTS@ ${DBUG}
 
 <sun4x_58 sunx86_58 sun4x_59 sunx86_59>
 LDFLAGS=-r -dy -N drv/ip -N drv/udp -N strmod/rpcmod