Configure glue for rxgk

Add an --enable-rxgk switch to control whether the feature is used.
For the sake of buildbot coverage, we still attempt to build the core
subdirectory provided that a sufficiently usable GSS-API library
is available, but do not install anything when rxgk is disabled at
configure time.  Future commits will use the configure argument to
control the behavior of other rxgk-aware code in the tree.

We provide a few new symbols to conditionally compile code for rxgk.
The two new high-level symbols are:

- AFS_RXGK_ENV: when defined, rxgk is available
- AFS_RXGK_GSS_ENV: when defined, we can use GSS-API calls

AFS_RXGK_GSS_ENV is turned on only for userspace pthread builds. For
now, AFS_RXGK_ENV is only turned on for userspace pthread builds, and
non-ukernel kernel builds. This effectively disables rxgk integration
in any ukernel or LWP code, but this can be changed in the future by
changing when AFS_RXGK_ENV is defined.

Change-Id: Iab661d47aac77c1a238e809362015b869752df18
Reviewed-on: https://gerrit.openafs.org/10564
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>
Reviewed-by: Andrew Deason <adeason@sinenomine.net>
Tested-by: BuildBot <buildbot@rampaginggeek.com>

Makefile.in

@@ -218,7 +218,12 @@ sys: cmd comerr afs hcrypto rx rxstat fsint sys_depinstall
 	+${COMPILE_PART1} sys ${COMPILE_PART2}
 
 rxgk: cmd comerr hcrypto rx rxgk_depinstall
-	+${COMPILE_PART1} rxgk ${COMPILE_PART2}
+	+set -x; \
+	if test "@BUILD_RXGK@" = "yes"; then \
+		${COMPILE_PART1} rxgk ${COMPILE_PART2} ; \
+	else \
+		echo Skipping rxgk for ${SYS_NAME} ; \
+	fi
 
 rxkad: cmd comerr hcrypto rfc3961 rx rxkad_depinstall
 	+${COMPILE_PART1} rxkad ${COMPILE_PART2}
@@ -633,7 +638,7 @@ finale: project cmd comerr afsd butc tbutc tbudb @ENABLE_KERNEL_MODULE@ \
 	bozo @VFSCK@ volser tvolser tsalvaged dviced dvolser libafscp \
 	venus venustests update xstat afsmonitor rxdebug libafsrpc rfc3961 \
 	hcrypto libafsauthent libadmin man-pages tools platform kopenafs \
-	authtools $(DIR_roken)
+	authtools $(DIR_roken) rxgk
 	+${COMPILE_PART1} finale ${COMPILE_PART2}
 
 finale_nolibafs: project cmd comerr afsd butc tbutc tbudb libuafs audit kauth \
@@ -641,7 +646,7 @@ finale_nolibafs: project cmd comerr afsd butc tbutc tbudb libuafs audit kauth \
 	volser tvolser tsalvaged dviced dvolser libafscp venus venustests \
 	update xstat afsmonitor rxdebug libafsrpc rfc3961 hcrypto \
 	libafsauthent libadmin man-pages tools platform kopenafs \
-	authtools $(DIR_roken)
+	authtools $(DIR_roken) rxgk
 	+${COMPILE_PART1} finale ${COMPILE_PART2}
 
 rpctestlib: util fsint rx opr libafsrpc libafsauthent

src/cf/gss.m4

@@ -9,8 +9,43 @@ dnl Check for the characteristics of whatever GSSAPI we found, if we found one
        AC_CHECK_FUNCS([gss_pseudo_random \
                        krb5_gss_register_acceptor_identity \
                        gss_krb5_ccache_name])
-dnl We only need gssapi for rxgk (at this point).  rxgk requires pseudo_random.
-       AS_IF([test x"$ac_cv_func_gss_pseudo_random" != xyes],
-             [BUILD_GSSAPI=no])
        RRA_LIB_GSSAPI_RESTORE])
-  AC_SUBST([BUILD_GSSAPI])])
+  AC_SUBST([BUILD_GSSAPI])
+
+dnl Determine if we should build rxgk
+  BUILD_RXGK=no
+  AS_IF([test x"$BUILD_GSSAPI" = xyes],
+    [BUILD_RXGK=yes
+dnl At this point, rxgk only needs gssapi in general, and gss_pseudo_random()
+     AS_IF([test x"$ac_cv_func_gss_pseudo_random" != xyes],
+       [BUILD_RXGK=no
+        AC_MSG_NOTICE([disabling rxgk due to unusable GSS-API library])])])
+  AC_SUBST([BUILD_RXGK])
+  AS_IF([test x"$BUILD_RXGK" = xyes],
+        [AC_DEFINE([BUILD_RXGK], [1], [Build rxgk])])
+
+dnl Determine if we should enable rxgk support (note that this is a different
+dnl decision than whether we should build rxgk)
+  ENABLE_RXGK="no"
+  RXGK_LIBS=""
+  RXGK_LIBS_RPC=""
+  RXGK_CFLAGS=""
+  RXGK_GSSAPI_LIBS=""
+  AS_IF([test "$enable_rxgk" = yes],
+    [AS_IF([test "$BUILD_RXGK" = yes],
+       [ENABLE_RXGK="yes"
+        RXGK_LIBS="\$(top_builddir)/src/rxgk/liboafs_rxgk.la"
+        RXGK_LIBS_RPC="\$(top_builddir)/src/rxgk/librxgk_pic.la"
+        RXGK_CFLAGS="\$(CPPFLAGS_gssapi)"
+        RXGK_GSSAPI_LIBS="\$(LDFLAGS_gssapi) \$(LIB_gssapi)"
+        AC_DEFINE([ENABLE_RXGK], [1],
+                  [Build rxgk support into applications])],
+
+       [AC_MSG_ERROR([Insufficient GSS-API support to enable rxgk])])],
+    [ENABLE_RXGK="no"])
+
+  AC_SUBST([ENABLE_RXGK])
+  AC_SUBST([RXGK_LIBS])
+  AC_SUBST([RXGK_LIBS_RPC])
+  AC_SUBST([RXGK_CFLAGS])
+  AC_SUBST([RXGK_GSSAPI_LIBS])])

src/cf/options.m4

@@ -107,6 +107,16 @@ AC_ARG_ENABLE([kauth],
     [enable_kauth="no"
      enable_pam="no"])
 
+dnl modern crypto
+AC_ARG_ENABLE([rxgk],
+dnl Note that this flag only dictates whether or not rxgk support is turned on
+dnl in various daemons, utilities, etc. Without this flag, we still always try
+dnl to build rxgk when we can (when a suitable gssapi is available).
+    [AS_HELP_STRING([--enable-rxgk],
+        [Include experimental support for the RXGK security class (defaults to disabled)])],
+    [],
+    [enable_rxgk="no"])
+
 dnl Optimization and debugging flags.
 AC_ARG_ENABLE([strip-binaries],
     [AS_HELP_STRING([--disable-strip-binaries],

src/config/Makefile.config.in

@@ -58,6 +58,7 @@ LD = @LD@
 LDFLAGS_roken = @LDFLAGS_roken@
 LDFLAGS_hcrypto = @LDFLAGS_hcrypto@
 LDFLAGS_krb5 = @KRB5_LDFLAGS@
+LDFLAGS_gssapi = @GSSAPI_LDFLAGS@
 CPPFLAGS_gssapi = @GSSAPI_CPPFLAGS@
 LEX = @LEX@
 LIB_AFSDB = @LIB_AFSDB@
@@ -97,6 +98,7 @@ REGEX_OBJ = @REGEX_OBJ@
 REGEX_OBJ_PIC = @REGEX_OBJ_PIC@
 RM = @RM@
 RXDEBUG = @RXDEBUG@
+RXGK_GSSAPI_LIBS = @RXGK_GSSAPI_LIBS@
 ROKEN_HEADERS = @ROKEN_HEADERS@
 ROKEN_LTLIBOBJS = @LTLIBOBJS@
 SED = @SED@

src/config/stds.h

@@ -238,6 +238,16 @@ typedef struct afsUUID afsUUID;
 #define AFS_DEMAND_ATTACH_FS 1
 #endif
 
+#if defined(BUILD_RXGK) && defined(AFS_PTHREAD_ENV) && !defined(KERNEL)
+/* Only build GSS bits of code on non-kernel pthread builds */
+# define AFS_RXGK_GSS_ENV 1
+#endif
+
+#if defined(ENABLE_RXGK) && !defined(UKERNEL) && (defined(AFS_PTHREAD_ENV) || defined(KERNEL))
+/* Only enable RXGK bits for userspace pthread or non-ukernel kernel code */
+# define AFS_RXGK_ENV 1
+#endif
+
 /* A macro that can be used when printf'ing 64 bit integers, as Unix and
  * windows use a different format string
  */

src/rxgk/Makefile.in

@@ -71,9 +71,21 @@ rxgk_errs.c: rxgk_errs.et
 #
 test: all
 
-install: liboafs_rxgk.la
+install: liboafs_rxgk.la rxgk.h rxgk_errs.h rxgk_int.h
+	if [ "@ENABLE_RXGK@" = yes ]; then \
+		${INSTALL} -d ${DESTDIR}${includedir}/rx; \
+		${INSTALL_DATA} ${srcdir}/rxgk.h ${DESTDIR}${includedir}/rx/rxgk.h ; \
+		${INSTALL_DATA} rxgk_errs.h ${DESTDIR}${includedir}/rx/rxgk_errs.h ; \
+		${INSTALL_DATA} rxgk_int.h ${DESTDIR}${includedir}/rx/rxgk_int.h ; \
+	fi
 
-dest: liboafs_rxgk.la
+dest: liboafs_rxgk.la rxgk.h rxgk_errs.h rxgk_int.h
+	if [ "@ENABLE_RXGK@" = yes ]; then \
+		${INSTALL} -d ${DESTDIR}${includedir}/rx; \
+		${INSTALL_DATA} ${srcdir}/rxgk.h ${DEST}/include/rx/rxgk.h ; \
+		${INSTALL_DATA} rxgk_errs.h ${DEST}/include/rx/rxgk_errs.h ; \
+		${INSTALL_DATA} rxgk_int.h ${DEST}/include/rx/rxgk_int.h ; \
+	fi
 
 #
 # Misc. targets

src/rxgk/rxgk_procs.c

@@ -37,6 +37,8 @@
 #include <afs/param.h>
 #include <afs/stds.h>
 
+#ifdef AFS_RXGK_GSS_ENV
+
 #include <rx/rx.h>
 #include <rx/rx_identity.h>
 #include <rx/rxgk.h>
@@ -70,3 +72,5 @@ SRXGK_AFSCombineTokens(struct rx_call *z_call, RXGK_Data *user_tok,
 {
     return RXGEN_OPCODE;
 }
+
+#endif /* AFS_RXGK_GSS_ENV */