arch: arm: Add support for multiple zero-latency irq priorities

Add the ability to have multiple irq priority levels which are not
masked by irq_lock() by adding CONFIG_ZERO_LATENCY_LEVELS.

If CONFIG_ZERO_LATENCY_LEVELS is set to a value > 1 then multiple zero
latency irqs are reserved by the kernel (and not only one). The priority
of the zero-latency interrupt can be configured by IRQ_CONNECT.

To be backwards compatible the prio argument in IRQ_CONNECT is still
ignored and the target prio set to zero if CONFIG_ZERO_LATENCY_LEVELS
is 1 (default).

Implements zephyrproject-rtos#45276

Signed-off-by: Christoph Coenen <[email protected]>

arch/arm/core/aarch32/cortex_m/Kconfig

@@ -302,6 +302,15 @@ config ZERO_LATENCY_IRQS
 	  higher priority than the rest of the kernel they cannot use any
 	  kernel functionality.
 
+config ZERO_LATENCY_LEVELS
+	int "Number of interrupt priority levels reserved for zero latency"
+	depends on ZERO_LATENCY_IRQS
+	range 1 255
+	help
+	  The amount of interrupt priority levels reserved for zero latency
+	  interrupts. Increase this value to reserve more than one priority
+	  level for zero latency interrupts.
+
 config DYNAMIC_DIRECT_INTERRUPTS
 	bool "Support for dynamic direct interrupts"
 	depends on DYNAMIC_INTERRUPTS

arch/arm/core/aarch32/irq_manage.c

@@ -74,7 +74,11 @@ void z_arm_irq_priority_set(unsigned int irq, unsigned int prio, uint32_t flags)
 	 * via flags
 	 */
 	if (IS_ENABLED(CONFIG_ZERO_LATENCY_IRQS) && (flags & IRQ_ZERO_LATENCY)) {
-		prio = _EXC_ZERO_LATENCY_IRQS_PRIO;
+		if (ZERO_LATENCY_LEVELS == 1) {
+			prio = _EXC_ZERO_LATENCY_IRQS_PRIO;
+		} else {
+			/* Use caller supplied prio level as-is */
+		}
 	} else {
 		prio += _IRQ_PRIO_OFFSET;
 	}

include/zephyr/arch/arm/aarch32/exc.h

@@ -49,7 +49,8 @@
 
 #define _EXC_FAULT_PRIO 0
 #define _EXC_ZERO_LATENCY_IRQS_PRIO 0
-#define _EXC_SVC_PRIO COND_CODE_1(CONFIG_ZERO_LATENCY_IRQS, (1), (0))
+#define _EXC_SVC_PRIO COND_CODE_1(CONFIG_ZERO_LATENCY_IRQS,		\
+				  (CONFIG_ZERO_LATENCY_LEVELS), (0))
 #define _IRQ_PRIO_OFFSET (_EXCEPTION_RESERVED_PRIO + _EXC_SVC_PRIO)
 #define IRQ_PRIO_LOWEST (BIT(NUM_IRQ_PRIO_BITS) - (_IRQ_PRIO_OFFSET) - 1)
 

include/zephyr/arch/arm/aarch32/irq.h

@@ -86,16 +86,27 @@ extern void z_arm_interrupt_init(void);
 
 /* Flags for use with IRQ_CONNECT() */
 /**
- * Set this interrupt up as a zero-latency IRQ. It has a fixed hardware
- * priority level (discarding what was supplied in the interrupt's priority
- * argument), and will run even if irq_lock() is active. Be careful!
+ * Set this interrupt up as a zero-latency IRQ. If CONFIG_ZERO_LATENCY_LEVELS
+ * is 1 it has a fixed hardware priority level (discarding what was supplied
+ * in the interrupt's priority argument). If CONFIG_ZERO_LATENCY_LEVELS is
+ * greater 1 it has the priority level assigned by the argument.
+ * The interrupt wil run even if irq_lock() is active. Be careful!
  */
 #define IRQ_ZERO_LATENCY	BIT(0)
 
 #ifdef CONFIG_CPU_CORTEX_M
+
+#if defined(CONFIG_ZERO_LATENCY_LEVELS)
+#define ZERO_LATENCY_LEVELS CONFIG_ZERO_LATENCY_LEVELS
+#else
+#define ZERO_LATENCY_LEVELS 1
+#endif
+
 #define _CHECK_PRIO(priority_p, flags_p) \
-	BUILD_ASSERT((flags_p & IRQ_ZERO_LATENCY) || \
-		     priority_p <= IRQ_PRIO_LOWEST, \
+	BUILD_ASSERT(((flags_p & IRQ_ZERO_LATENCY) && \
+		      ((ZERO_LATENCY_LEVELS == 1) || \
+		       (priority_p < ZERO_LATENCY_LEVELS))) || \
+		     (priority_p <= IRQ_PRIO_LOWEST), \
 		     "Invalid interrupt priority. Values must not exceed IRQ_PRIO_LOWEST");
 #else
 #define _CHECK_PRIO(priority_p, flags_p)

tests/arch/arm/arm_irq_zero_latency_levels/CMakeLists.txt

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(arm_irq_zero_latency_levels)
+
+target_sources(app PRIVATE
+  src/main.c
+)

tests/arch/arm/arm_irq_zero_latency_levels/prj.conf

@@ -0,0 +1,5 @@
+CONFIG_ZTEST=y
+CONFIG_DYNAMIC_INTERRUPTS=y
+CONFIG_DYNAMIC_DIRECT_INTERRUPTS=y
+CONFIG_ZERO_LATENCY_IRQS=y
+CONFIG_ZERO_LATENCY_LEVELS=2

tests/arch/arm/arm_irq_zero_latency_levels/src/main.c

@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2022 Baumer (www.baumer.com)
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <ztest.h>
+#include <arch/cpu.h>
+#include <arch/arm/aarch32/cortex_m/cmsis.h>
+
+
+#define EXECUTION_TRACE_LENGTH 6
+
+#define IRQ_A_PRIO  1   /* lower priority */
+#define IRQ_B_PRIO  0   /* higher priority */
+
+
+#define CHECK_STEP(pos, val) zassert_equal(	\
+	execution_trace[pos],			\
+	val,					\
+	"Expected %s for step %d but got %s",	\
+	execution_step_str(val),		\
+	pos,					\
+	execution_step_str(execution_trace[pos]))
+
+
+enum execution_step {
+	STEP_MAIN_BEGIN,
+	STEP_MAIN_END,
+	STEP_ISR_A_BEGIN,
+	STEP_ISR_A_END,
+	STEP_ISR_B_BEGIN,
+	STEP_ISR_B_END,
+};
+
+static volatile enum execution_step execution_trace[EXECUTION_TRACE_LENGTH];
+static volatile int execution_trace_pos;
+
+static int irq_a;
+static int irq_b;
+
+static const char *execution_step_str(enum execution_step s)
+{
+	const char *res = "invalid";
+
+	switch (s) {
+	case STEP_MAIN_BEGIN:
+		res = "STEP_MAIN_BEGIN";
+		break;
+	case STEP_MAIN_END:
+		res = "STEP_MAIN_END";
+		break;
+	case STEP_ISR_A_BEGIN:
+		res = "STEP_ISR_A_BEGIN";
+		break;
+	case STEP_ISR_A_END:
+		res = "STEP_ISR_A_END";
+		break;
+	case STEP_ISR_B_BEGIN:
+		res = "STEP_ISR_B_BEGIN";
+		break;
+	case STEP_ISR_B_END:
+		res = "STEP_ISR_B_END";
+		break;
+	default:
+		break;
+	}
+	return res;
+}
+
+
+static void execution_trace_add(enum execution_step s)
+{
+	__ASSERT(execution_trace_pos < EXECUTION_TRACE_LENGTH,
+		 "Execution trace overflow");
+	execution_trace[execution_trace_pos] = s;
+	execution_trace_pos++;
+}
+
+
+
+void isr_a_handler(const void *args)
+{
+	ARG_UNUSED(args);
+	execution_trace_add(STEP_ISR_A_BEGIN);
+
+	/* Set higher prior irq b pending */
+	NVIC_SetPendingIRQ(irq_b);
+	__DSB();
+	__ISB();
+
+	execution_trace_add(STEP_ISR_A_END);
+}
+
+
+void isr_b_handler(const void *args)
+{
+	ARG_UNUSED(args);
+	execution_trace_add(STEP_ISR_B_BEGIN);
+	execution_trace_add(STEP_ISR_B_END);
+}
+
+
+static int find_unused_irq(int start)
+{
+	int i;
+
+	for (i = start - 1; i >= 0; i--) {
+		if (NVIC_GetEnableIRQ(i) == 0) {
+			/*
+			 * Interrupts configured statically with IRQ_CONNECT(.)
+			 * are automatically enabled. NVIC_GetEnableIRQ()
+			 * returning false, here, implies that the IRQ line is
+			 * either not implemented or it is not enabled, thus,
+			 * currently not in use by Zephyr.
+			 */
+
+			/* Set the NVIC line to pending. */
+			NVIC_SetPendingIRQ(i);
+
+			if (NVIC_GetPendingIRQ(i)) {
+				/*
+				 * If the NVIC line is pending, it is
+				 * guaranteed that it is implemented; clear the
+				 * line.
+				 */
+				NVIC_ClearPendingIRQ(i);
+
+				if (!NVIC_GetPendingIRQ(i)) {
+					/*
+					 * If the NVIC line can be successfully
+					 * un-pended, it is guaranteed that it
+					 * can be used for software interrupt
+					 * triggering. Return the NVIC line
+					 * number.
+					 */
+					break;
+				}
+			}
+		}
+	}
+
+	zassert_true(i >= 0,
+		     "No available IRQ line to configure as zero-latency\n");
+
+	TC_PRINT("Available IRQ line: %u\n", i);
+	return i;
+}
+
+
+void test_arm_zero_latency_levels(void)
+{
+	/*
+	 * Confirm that a zero-latency interrupt with lower priority will be
+	 * interrupted by a zero-latency interrupt with higher priority.
+	 */
+
+	if (!IS_ENABLED(CONFIG_ZERO_LATENCY_IRQS)) {
+		TC_PRINT("Skipped (Cortex-M Mainline only)\n");
+		return;
+	}
+
+	/* Determine two NVIC IRQ lines that are not currently in use. */
+	irq_a = find_unused_irq(CONFIG_NUM_IRQS);
+	irq_b = find_unused_irq(irq_a);
+
+	/* Configure IRQ A as zero-latency interrupt with prio 1 */
+	arch_irq_connect_dynamic(irq_a, IRQ_A_PRIO, isr_a_handler,
+				 NULL, IRQ_ZERO_LATENCY);
+	NVIC_ClearPendingIRQ(irq_a);
+	NVIC_EnableIRQ(irq_a);
+
+	/* Configure irq_b as zero-latency interrupt with prio 0 */
+	arch_irq_connect_dynamic(irq_b, IRQ_B_PRIO, isr_b_handler,
+				 NULL, IRQ_ZERO_LATENCY);
+	NVIC_ClearPendingIRQ(irq_b);
+	NVIC_EnableIRQ(irq_b);
+
+	/* Lock interrupts */
+	int key = irq_lock();
+
+	execution_trace_add(STEP_MAIN_BEGIN);
+
+	/* Trigger irq_a */
+	NVIC_SetPendingIRQ(irq_a);
+	__DSB();
+	__ISB();
+
+	execution_trace_add(STEP_MAIN_END);
+
+	/* Confirm that irq_a interrupted main and irq_b interrupted irq_a */
+	CHECK_STEP(0, STEP_MAIN_BEGIN);
+	CHECK_STEP(1, STEP_ISR_A_BEGIN);
+	CHECK_STEP(2, STEP_ISR_B_BEGIN);
+	CHECK_STEP(3, STEP_ISR_B_END);
+	CHECK_STEP(4, STEP_ISR_A_END);
+	CHECK_STEP(5, STEP_MAIN_END);
+
+	/* Unlock interrupts */
+	irq_unlock(key);
+}
+
+
+void test_main(void)
+{
+	ztest_test_suite(arm_irq_zero_latency_levels,
+		ztest_unit_test(test_arm_zero_latency_levels));
+	ztest_run_test_suite(arm_irq_zero_latency_levels);
+}

tests/arch/arm/arm_irq_zero_latency_levels/testcase.yaml

@@ -0,0 +1,9 @@
+common:
+  filter: (CONFIG_ARMV6_M_ARMV8_M_BASELINE or CONFIG_ARMV7_M_ARMV8_M_MAINLINE) and not CONFIG_SOC_FAMILY_NRF
+  tags: arm interrupt
+  arch_allow: arm
+tests:
+  arch.arm.irq_zero_latency_levels:
+    filter: not CONFIG_TRUSTED_EXECUTION_NONSECURE
+  arch.arm.irq_zero_latency_levels.secure_fw:
+    filter: CONFIG_TRUSTED_EXECUTION_SECURE