arm: apple: nvme: Add SART support and RTKit buffer management

The NVMe firmware in the macOS 13 beta blocks or crashes with u-boot's
current minimal RTKit implementation. It does not provide buffers for
the firmware's buffer requests. The ANS2 firmware included in macOS 11
and 12 tolerates this. The firmware included in the first macOS 13 beta
requires buffers for the crashlog and ioreport endpoints to function.

In the case of the NVMe the buffers are physical memory. Access to
physical memory is guarded by what Apple calls SART.
Import m1n1's SART driver (exclusively used for the NVMe controller).
Implement buffer management helpers for RTKit. These are generic since
other devices (none in u-boot so far) require different handling.

Signed-off-by: Janne Grunau <j@jannau.net>
Reviewed-by: Mark Kettenis <kettenis@openbsd.org>
Tested-by: Mark Kettenis <kettenis@openbsd.org>

5 files changed, 406 insertions, 28 deletions

diff --git a/arch/arm/include/asm/arch-apple/rtkit.h b/arch/arm/include/asm/arch-apple/rtkit.h
index 51f77f298c..eff18ddb9d 100644
--- a/arch/arm/include/asm/arch-apple/rtkit.h
+++ b/arch/arm/include/asm/arch-apple/rtkit.h
@@ -7,5 +7,23 @@
 #define APPLE_RTKIT_PWR_STATE_QUIESCED	0x10
 #define APPLE_RTKIT_PWR_STATE_ON	0x20
 
-int apple_rtkit_init(struct mbox_chan *);
-int apple_rtkit_shutdown(struct mbox_chan *, int);
+struct apple_rtkit_buffer {
+	void *buffer;
+	u64 dva;
+	size_t size;
+	bool is_mapped;
+};
+
+typedef int (*apple_rtkit_shmem_setup)(void *cookie,
+				       struct apple_rtkit_buffer *buf);
+typedef void (*apple_rtkit_shmem_destroy)(void *cookie,
+					  struct apple_rtkit_buffer *buf);
+
+struct apple_rtkit;
+
+struct apple_rtkit *apple_rtkit_init(struct mbox_chan *chan, void *cookie,
+				     apple_rtkit_shmem_setup shmem_setup,
+				     apple_rtkit_shmem_destroy shmem_destroy);
+void apple_rtkit_free(struct apple_rtkit *rtk);
+int apple_rtkit_boot(struct apple_rtkit *rtk);
+int apple_rtkit_shutdown(struct apple_rtkit *rtk, int pwrstate);
diff --git a/arch/arm/include/asm/arch-apple/sart.h b/arch/arm/include/asm/arch-apple/sart.h
new file mode 100644
index 0000000000..b99bdeafc0
--- /dev/null
+++ b/arch/arm/include/asm/arch-apple/sart.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * The sart code is copied from m1n1 (https://github.com/AsahiLinux/m1n1) and
+ * licensed as MIT.
+ *
+ * (C) Copyright 2022 The Asahi Linux Contributors
+ */
+
+#ifndef SART_H
+#define SART_H
+
+#include <dm/ofnode.h>
+
+struct apple_sart;
+
+struct apple_sart *sart_init(ofnode node);
+void sart_free(struct apple_sart *sart);
+
+bool sart_add_allowed_region(struct apple_sart *sart, void *paddr, size_t sz);
+bool sart_remove_allowed_region(struct apple_sart *sart, void *paddr, size_t sz);
+
+#endif
diff --git a/arch/arm/mach-apple/Makefile b/arch/arm/mach-apple/Makefile
index 52f30a777b..50b465b947 100644
--- a/arch/arm/mach-apple/Makefile
+++ b/arch/arm/mach-apple/Makefile
@@ -3,3 +3,4 @@
 obj-y += board.o
 obj-y += lowlevel_init.o
 obj-y += rtkit.o
+obj-$(CONFIG_NVME_APPLE) += sart.o
diff --git a/arch/arm/mach-apple/rtkit.c b/arch/arm/mach-apple/rtkit.c
index 2dcb8bdd3e..da77718442 100644
--- a/arch/arm/mach-apple/rtkit.c
+++ b/arch/arm/mach-apple/rtkit.c
@@ -17,6 +17,7 @@
 #define APPLE_RTKIT_EP_SYSLOG 2
 #define APPLE_RTKIT_EP_DEBUG 3
 #define APPLE_RTKIT_EP_IOREPORT 4
+#define APPLE_RTKIT_EP_TRACEKIT 10
 
 /* Messages for management endpoint. */
 #define APPLE_RTKIT_MGMT_TYPE GENMASK(59, 52)
@@ -51,7 +52,102 @@
 #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK(51, 44)
 #define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK(41, 0)
 
-int apple_rtkit_init(struct mbox_chan *chan)
+struct apple_rtkit {
+	struct mbox_chan *chan;
+	void *cookie;
+	apple_rtkit_shmem_setup shmem_setup;
+	apple_rtkit_shmem_destroy shmem_destroy;
+
+	struct apple_rtkit_buffer syslog_buffer;
+	struct apple_rtkit_buffer crashlog_buffer;
+	struct apple_rtkit_buffer ioreport_buffer;
+};
+
+struct apple_rtkit *apple_rtkit_init(struct mbox_chan *chan, void *cookie,
+				     apple_rtkit_shmem_setup shmem_setup,
+				     apple_rtkit_shmem_destroy shmem_destroy)
+{
+	struct apple_rtkit *rtk;
+
+	rtk = calloc(sizeof(*rtk), 1);
+	if (!rtk)
+		return NULL;
+
+	rtk->chan = chan;
+	rtk->cookie = cookie;
+	rtk->shmem_setup = shmem_setup;
+	rtk->shmem_destroy = shmem_destroy;
+
+	return rtk;
+}
+
+void apple_rtkit_free(struct apple_rtkit *rtk)
+{
+	if (rtk->shmem_destroy) {
+		if (rtk->syslog_buffer.buffer)
+			rtk->shmem_destroy(rtk->cookie, &rtk->syslog_buffer);
+		if (rtk->crashlog_buffer.buffer)
+			rtk->shmem_destroy(rtk->cookie, &rtk->crashlog_buffer);
+		if (rtk->ioreport_buffer.buffer)
+			rtk->shmem_destroy(rtk->cookie, &rtk->ioreport_buffer);
+	}
+	free(rtk);
+}
+
+static int rtkit_handle_buf_req(struct apple_rtkit *rtk, int endpoint, struct apple_mbox_msg *msg)
+{
+	struct apple_rtkit_buffer *buf;
+	size_t num_4kpages;
+	int ret;
+
+	num_4kpages = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg->msg0);
+
+	if (num_4kpages == 0) {
+		printf("%s: unexpected request for buffer without size\n", __func__);
+		return -1;
+	}
+
+	switch (endpoint) {
+	case APPLE_RTKIT_EP_CRASHLOG:
+		buf = &rtk->crashlog_buffer;
+		break;
+	case APPLE_RTKIT_EP_SYSLOG:
+		buf = &rtk->syslog_buffer;
+		break;
+	case APPLE_RTKIT_EP_IOREPORT:
+		buf = &rtk->ioreport_buffer;
+		break;
+	default:
+		printf("%s: unexpected endpoint %d\n", __func__, endpoint);
+		return -1;
+	}
+
+	buf->dva = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg->msg0);
+	buf->size = num_4kpages << 12;
+	buf->is_mapped = false;
+
+	if (rtk->shmem_setup) {
+		ret = rtk->shmem_setup(rtk->cookie, buf);
+		if (ret < 0) {
+			printf("%s: shmen_setup failed for endpoint %d\n", __func__,
+			       endpoint);
+			return ret;
+		}
+	}
+
+	if (!buf->is_mapped) {
+		msg->msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_BUFFER_REQUEST) |
+				FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, num_4kpages) |
+				FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, buf->dva);
+		msg->msg1 = endpoint;
+
+		return mbox_send(rtk->chan, msg);
+	}
+
+	return 0;
+}
+
+int apple_rtkit_boot(struct apple_rtkit *rtk)
 {
 	struct apple_mbox_msg msg;
 	int endpoints[256];
@@ -67,12 +163,12 @@ int apple_rtkit_init(struct mbox_chan *chan)
 	msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE) |
 		FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_ON);
 	msg.msg1 = APPLE_RTKIT_EP_MGMT;
-	ret = mbox_send(chan, &msg);
+	ret = mbox_send(rtk->chan, &msg);
 	if (ret < 0)
 		return ret;
 
 	/* Wait for protocol version negotiation message. */
-	ret = mbox_recv(chan, &msg, 10000);
+	ret = mbox_recv(rtk->chan, &msg, 10000);
 	if (ret < 0)
 		return ret;
 
@@ -108,13 +204,13 @@ int apple_rtkit_init(struct mbox_chan *chan)
 		FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver) |
 		FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver);
 	msg.msg1 = APPLE_RTKIT_EP_MGMT;
-	ret = mbox_send(chan, &msg);
+	ret = mbox_send(rtk->chan, &msg);
 	if (ret < 0)
 		return ret;
 
 wait_epmap:
 	/* Wait for endpoint map message. */
-	ret = mbox_recv(chan, &msg, 10000);
+	ret = mbox_recv(rtk->chan, &msg, 10000);
 	if (ret < 0)
 		return ret;
 
@@ -145,7 +241,7 @@ wait_epmap:
 		reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE;
 	msg.msg0 = reply;
 	msg.msg1 = APPLE_RTKIT_EP_MGMT;
-	ret = mbox_send(chan, &msg);
+	ret = mbox_send(rtk->chan, &msg);
 	if (ret < 0)
 		return ret;
 
@@ -153,24 +249,33 @@ wait_epmap:
 		goto wait_epmap;
 
 	for (i = 0; i < nendpoints; i++) {
-		/* Don't start the syslog endpoint since we can't
-		   easily handle its messages in U-Boot. */
-		if (endpoints[i] == APPLE_RTKIT_EP_SYSLOG)
+		/* Start only necessary endpoints. The syslog endpoint is
+		 * particularly noisy and its message can't easily be handled
+		 * within U-Boot.
+		 */
+		switch (endpoints[i]) {
+		case APPLE_RTKIT_EP_MGMT:
+		case APPLE_RTKIT_EP_SYSLOG:
+		case APPLE_RTKIT_EP_DEBUG:
+		case APPLE_RTKIT_EP_TRACEKIT:
 			continue;
+		default:
+			break;
+		}
 
 		/* Request endpoint. */
 		msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_STARTEP) |
 			FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoints[i]) |
 			APPLE_RTKIT_MGMT_STARTEP_FLAG;
 		msg.msg1 = APPLE_RTKIT_EP_MGMT;
-		ret = mbox_send(chan, &msg);
+		ret = mbox_send(rtk->chan, &msg);
 		if (ret < 0)
 			return ret;
 	}
 
 	pwrstate = APPLE_RTKIT_PWR_STATE_SLEEP;
 	while (pwrstate != APPLE_RTKIT_PWR_STATE_ON) {
-		ret = mbox_recv(chan, &msg, 1000000);
+		ret = mbox_recv(rtk->chan, &msg, 100000);
 		if (ret < 0)
 			return ret;
 
@@ -180,20 +285,22 @@ wait_epmap:
 		if (endpoint == APPLE_RTKIT_EP_CRASHLOG ||
 		    endpoint == APPLE_RTKIT_EP_SYSLOG ||
 		    endpoint == APPLE_RTKIT_EP_IOREPORT) {
-			u64 addr = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg.msg0);
-			u64 size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg.msg0);
-
-			if (msgtype == APPLE_RTKIT_BUFFER_REQUEST && addr != 0)
+			if (msgtype == APPLE_RTKIT_BUFFER_REQUEST) {
+				ret = rtkit_handle_buf_req(rtk, endpoint, &msg);
+				if (ret < 0)
+					return ret;
 				continue;
+			}
+		}
 
-			msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_BUFFER_REQUEST) |
-				FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, size) |
-				FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, addr);
-			msg.msg1 = endpoint;
-			ret = mbox_send(chan, &msg);
-			if (ret < 0)
-				return ret;
-			continue;
+		if (endpoint == APPLE_RTKIT_EP_IOREPORT) {
+			// these two messages have to be ack-ed for proper startup
+			if (msgtype == 0xc || msgtype == 0x8) {
+				ret = mbox_send(rtk->chan, &msg);
+				if (ret < 0)
+					return ret;
+				continue;
+			}
 		}
 
 		if (endpoint != APPLE_RTKIT_EP_MGMT) {
@@ -211,7 +318,7 @@ wait_epmap:
 	return 0;
 }
 
-int apple_rtkit_shutdown(struct mbox_chan *chan, int pwrstate)
+int apple_rtkit_shutdown(struct apple_rtkit *rtk, int pwrstate)
 {
 	struct apple_mbox_msg msg;
 	int ret;
@@ -219,11 +326,11 @@ int apple_rtkit_shutdown(struct mbox_chan *chan, int pwrstate)
 	msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE) |
 		FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, pwrstate);
 	msg.msg1 = APPLE_RTKIT_EP_MGMT;
-	ret = mbox_send(chan, &msg);
+	ret = mbox_send(rtk->chan, &msg);
 	if (ret < 0)
 		return ret;
 
-	ret = mbox_recv(chan, &msg, 100000);
+	ret = mbox_recv(rtk->chan, &msg, 100000);
 	if (ret < 0)
 		return ret;
 
diff --git a/arch/arm/mach-apple/sart.c b/arch/arm/mach-apple/sart.c
new file mode 100644
index 0000000000..e9b017ad57
--- /dev/null
+++ b/arch/arm/mach-apple/sart.c
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: MIT
+/*
+ * The sart code is copied from m1n1 (https://github.com/AsahiLinux/m1n1) and
+ * licensed as MIT.
+ *
+ * (C) Copyright 2022 The Asahi Linux Contributors
+ */
+
+#include <asm/io.h>
+#include <asm/arch/sart.h>
+
+#include <linux/bitfield.h>
+#include <linux/types.h>
+
+#include <malloc.h>
+
+#define APPLE_SART_MAX_ENTRIES 16
+
+/* This is probably a bitfield but the exact meaning of each bit is unknown. */
+#define APPLE_SART_FLAGS_ALLOW 0xff
+
+/* SARTv2 registers */
+#define APPLE_SART2_CONFIG(idx)       (0x00 + 4 * (idx))
+#define APPLE_SART2_CONFIG_FLAGS      GENMASK(31, 24)
+#define APPLE_SART2_CONFIG_SIZE       GENMASK(23, 0)
+#define APPLE_SART2_CONFIG_SIZE_SHIFT 12
+#define APPLE_SART2_CONFIG_SIZE_MAX   GENMASK(23, 0)
+
+#define APPLE_SART2_PADDR(idx)  (0x40 + 4 * (idx))
+#define APPLE_SART2_PADDR_SHIFT 12
+
+/* SARTv3 registers */
+#define APPLE_SART3_CONFIG(idx) (0x00 + 4 * (idx))
+
+#define APPLE_SART3_PADDR(idx)  (0x40 + 4 * (idx))
+#define APPLE_SART3_PADDR_SHIFT 12
+
+#define APPLE_SART3_SIZE(idx)  (0x80 + 4 * (idx))
+#define APPLE_SART3_SIZE_SHIFT 12
+#define APPLE_SART3_SIZE_MAX   GENMASK(29, 0)
+
+struct apple_sart {
+	uintptr_t base;
+	u32 protected_entries;
+
+	void (*get_entry)(struct apple_sart *sart, int index, u8 *flags, void **paddr,
+			  size_t *size);
+	bool (*set_entry)(struct apple_sart *sart, int index, u8 flags, void *paddr,
+			  size_t size);
+};
+
+static void sart2_get_entry(struct apple_sart *sart, int index, u8 *flags, void **paddr,
+			    size_t *size)
+{
+	u32 cfg = readl(sart->base + APPLE_SART2_CONFIG(index));
+	*flags = FIELD_GET(APPLE_SART2_CONFIG_FLAGS, cfg);
+	*size = (size_t)FIELD_GET(APPLE_SART2_CONFIG_SIZE, cfg) << APPLE_SART2_CONFIG_SIZE_SHIFT;
+	*paddr = (void *)
+		((u64)readl(sart->base + APPLE_SART2_PADDR(index)) << APPLE_SART2_PADDR_SHIFT);
+}
+
+static bool sart2_set_entry(struct apple_sart *sart, int index, u8 flags, void *paddr_,
+			    size_t size)
+{
+	u32 cfg;
+	u64 paddr = (u64)paddr_;
+
+	if (size & ((1 << APPLE_SART2_CONFIG_SIZE_SHIFT) - 1))
+		return false;
+	if (paddr & ((1 << APPLE_SART2_PADDR_SHIFT) - 1))
+		return false;
+
+	size >>= APPLE_SART2_CONFIG_SIZE_SHIFT;
+	paddr >>= APPLE_SART2_PADDR_SHIFT;
+
+	if (size > APPLE_SART2_CONFIG_SIZE_MAX)
+		return false;
+
+	cfg = FIELD_PREP(APPLE_SART2_CONFIG_FLAGS, flags);
+	cfg |= FIELD_PREP(APPLE_SART2_CONFIG_SIZE, size);
+
+	writel(paddr, sart->base + APPLE_SART2_PADDR(index));
+	writel(cfg, sart->base + APPLE_SART2_CONFIG(index));
+
+	return true;
+}
+
+static void sart3_get_entry(struct apple_sart *sart, int index, u8 *flags, void **paddr,
+			    size_t *size)
+{
+	*flags = readl(sart->base + APPLE_SART3_CONFIG(index));
+	*size = (size_t)readl(sart->base + APPLE_SART3_SIZE(index)) << APPLE_SART3_SIZE_SHIFT;
+	*paddr = (void *)
+		((u64)readl(sart->base + APPLE_SART3_PADDR(index)) << APPLE_SART3_PADDR_SHIFT);
+}
+
+static bool sart3_set_entry(struct apple_sart *sart, int index, u8 flags, void *paddr_,
+			    size_t size)
+{
+	u64 paddr = (u64)paddr_;
+
+	if (size & ((1 << APPLE_SART3_SIZE_SHIFT) - 1))
+		return false;
+	if (paddr & ((1 << APPLE_SART3_PADDR_SHIFT) - 1))
+		return false;
+
+	paddr >>= APPLE_SART3_PADDR_SHIFT;
+	size >>= APPLE_SART3_SIZE_SHIFT;
+
+	if (size > APPLE_SART3_SIZE_MAX)
+		return false;
+
+	writel(paddr, sart->base + APPLE_SART3_PADDR(index));
+	writel(size, sart->base + APPLE_SART3_SIZE(index));
+	writel(flags, sart->base + APPLE_SART3_CONFIG(index));
+
+	return true;
+}
+
+struct apple_sart *sart_init(ofnode node)
+{
+	phys_addr_t base;
+	u32 sart_version;
+	struct apple_sart *sart;
+
+	base = ofnode_get_addr(node);
+	if (base == FDT_ADDR_T_NONE)
+		return NULL;
+
+	if (ofnode_device_is_compatible(node, "apple,t8103-sart")) {
+		sart_version = 2;
+	} else if (ofnode_device_is_compatible(node, "apple,t6000-sart")) {
+		sart_version = 3;
+	} else {
+		printf("sart: unknown SART compatible: %sd\n",
+		       ofnode_read_string(node, "compatible"));
+		return NULL;
+	}
+
+	sart = calloc(sizeof(*sart), 1);
+	if (!sart)
+		return NULL;
+
+	sart->base = base;
+
+	switch (sart_version) {
+	case 2:
+		sart->get_entry = sart2_get_entry;
+		sart->set_entry = sart2_set_entry;
+		break;
+	case 3:
+		sart->get_entry = sart3_get_entry;
+		sart->set_entry = sart3_set_entry;
+		break;
+	default:
+		printf("sart: SART has unknown version %d\n", sart_version);
+		free(sart);
+		return NULL;
+	}
+
+	sart->protected_entries = 0;
+	for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {
+		void *paddr;
+		u8 flags;
+		size_t sz;
+
+		sart->get_entry(sart, i, &flags, &paddr, &sz);
+		if (flags)
+			sart->protected_entries |= 1 << i;
+	}
+
+	return sart;
+}
+
+void sart_free(struct apple_sart *sart)
+{
+	for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {
+		if (sart->protected_entries & (1 << i))
+			continue;
+		sart->set_entry(sart, i, 0, NULL, 0);
+	}
+
+	free(sart);
+}
+
+bool sart_add_allowed_region(struct apple_sart *sart, void *paddr, size_t sz)
+{
+	for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {
+		void *e_paddr;
+		u8 e_flags;
+		size_t e_sz;
+
+		if (sart->protected_entries & (1 << i))
+			continue;
+
+		sart->get_entry(sart, i, &e_flags, &e_paddr, &e_sz);
+		if (e_flags)
+			continue;
+
+		return sart->set_entry(sart, i, APPLE_SART_FLAGS_ALLOW, paddr, sz);
+	}
+
+	printf("sart: no more free entries\n");
+	return false;
+}
+
+bool sart_remove_allowed_region(struct apple_sart *sart, void *paddr, size_t sz)
+{
+	for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {
+		void *e_paddr;
+		u8 e_flags;
+		size_t e_sz;
+
+		if (sart->protected_entries & (1 << i))
+			continue;
+
+		sart->get_entry(sart, i, &e_flags, &e_paddr, &e_sz);
+		if (!e_flags)
+			continue;
+		if (e_paddr != paddr)
+			continue;
+		if (e_sz != sz)
+			continue;
+
+		return sart->set_entry(sart, i, 0, NULL, 0);
+	}
+
+	printf("sart: could not find entry to be removed\n");
+	return false;
+}