riscv: dcache flush hooks

Cache management operations were, for a long time, unspecified by the
RISC-V ISA, and thus these functions have been no-ops. To cope, hardware
with non-coherent I/O has implemented custom cache flush mechanisms,
either in the form of custom instructions or special device registers.
Additionally, the RISC-V CMO extension is ratified and these official
instructions will start to show up in hardware eventually. Therefore, a
method is needed to select the dcache management routines at runtime.

Add a simple set of function hooks, as well as a routine to install them
and specify the minimum dcache line size. The first consumer will be the
non-standard cache management instructions for T-HEAD CPUs.

The unused I-cache variables and macros are removed.

Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D47454

sys/conf/files.riscv

@@ -36,6 +36,7 @@ riscv/riscv/bus_machdep.c	standard
 riscv/riscv/bus_space_asm.S	standard
 riscv/riscv/busdma_bounce.c	standard
 riscv/riscv/busdma_machdep.c	standard
+riscv/riscv/cache.c		standard
 riscv/riscv/clock.c		standard
 riscv/riscv/copyinout.S		standard
 riscv/riscv/cpufunc_asm.S	standard

sys/riscv/include/cpufunc.h

@@ -44,6 +44,8 @@ breakpoint(void)
 
 #ifdef _KERNEL
 
+#include <sys/_null.h>
+
 #include <machine/riscvreg.h>
 
 static __inline register_t
@@ -107,16 +109,42 @@ sfence_vma_page(uintptr_t addr)
 #define	rdinstret()			csr_read64(instret)
 #define	rdhpmcounter(n)			csr_read64(hpmcounter##n)
 
+/* Cache hooks. */
+
 extern int64_t dcache_line_size;
-extern int64_t icache_line_size;
 
-#define	cpu_dcache_wbinv_range(a, s)
-#define	cpu_dcache_inv_range(a, s)
-#define	cpu_dcache_wb_range(a, s)
+typedef void (*cache_op_t)(vm_offset_t start, vm_size_t size);
 
-#define	cpu_idcache_wbinv_range(a, s)
-#define	cpu_icache_sync_range(a, s)
-#define	cpu_icache_sync_range_checked(a, s)
+struct riscv_cache_ops {
+	cache_op_t dcache_wbinv_range;
+	cache_op_t dcache_inv_range;
+	cache_op_t dcache_wb_range;
+};
+
+extern struct riscv_cache_ops cache_ops;
+
+static __inline void
+cpu_dcache_wbinv_range(vm_offset_t addr, vm_size_t size)
+{
+	if (cache_ops.dcache_wbinv_range != NULL)
+		cache_ops.dcache_wbinv_range(addr, size);
+}
+
+static __inline void
+cpu_dcache_inv_range(vm_offset_t addr, vm_size_t size)
+{
+	if (cache_ops.dcache_inv_range != NULL)
+		cache_ops.dcache_inv_range(addr, size);
+}
+
+static __inline void
+cpu_dcache_wb_range(vm_offset_t addr, vm_size_t size)
+{
+	if (cache_ops.dcache_wb_range != NULL)
+		cache_ops.dcache_wb_range(addr, size);
+}
+
+void riscv_cache_install_hooks(struct riscv_cache_ops *, u_int);
 
 #define	cpufunc_nullop()		riscv_nullop()
 

sys/riscv/riscv/cache.c

@@ -0,0 +1,53 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 The FreeBSD Foundation
+ *
+ * This software was developed by Mitchell Horne <mhorne@FreeBSD.org> under
+ * sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+
+#include <machine/cpufunc.h>
+
+struct riscv_cache_ops __read_frequently cache_ops;
+
+int64_t __read_frequently dcache_line_size;	/* The minimum D cache line size */
+
+static bool cache_initialized;
+
+void
+riscv_cache_install_hooks(struct riscv_cache_ops *newops, u_int line_size)
+{
+	if (cache_initialized)
+		panic("cache hooks already installed!");
+
+	bcopy(newops, &cache_ops, sizeof(cache_ops));
+	dcache_line_size = line_size;
+
+	cache_initialized = true;
+}

sys/riscv/riscv/machdep.c

@@ -113,10 +113,6 @@ int cold = 1;
 
 struct kva_md_info kmi;
 
-int64_t dcache_line_size;	/* The minimum D cache line size */
-int64_t icache_line_size;	/* The minimum I cache line size */
-int64_t idcache_line_size;	/* The minimum cache line size */
-
 #define BOOT_HART_INVALID	0xffffffff
 uint32_t boot_hart = BOOT_HART_INVALID;	/* The hart we booted on. */
 
@@ -329,17 +325,6 @@ try_load_dtb(caddr_t kmdp)
 }
 #endif
 
-static void
-cache_setup(void)
-{
-
-	/* TODO */
-
-	dcache_line_size = 0;
-	icache_line_size = 0;
-	idcache_line_size = 0;
-}
-
 /*
  * Fake up a boot descriptor table.
  */
@@ -550,8 +535,6 @@ initriscv(struct riscv_bootparams *rvbp)
 	/* Do basic tuning, hz etc */
 	init_param1();
 
-	cache_setup();
-
 #ifdef FDT
 	/*
 	 * XXX: Unconditionally exclude the lowest 2MB of physical memory, as

sys/riscv/riscv/pmap.c

@@ -5013,7 +5013,7 @@ pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode)
 	if (anychanged) {
 		pmap_invalidate_range(kernel_pmap, base, tmpva);
 		if (mode == VM_MEMATTR_UNCACHEABLE)
-			cpu_dcache_wbinv_range((void *)base, size);
+			cpu_dcache_wbinv_range(base, size);
 	}
 
 	return (error);