schedule: add Tasks With Budget scheduler type

app/boards/intel_adsp_ace15_mtpm.conf

@@ -48,6 +48,7 @@ CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y
 CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y
 CONFIG_ZEPHYR_NATIVE_DRIVERS=y
 CONFIG_ZEPHYR_DP_SCHEDULER=y
+CONFIG_ZEPHYR_TWB_SCHEDULER=y
 CONFIG_COLD_STORE_EXECUTE_DRAM=y
 
 # SOF / loadable modules
@@ -70,6 +71,9 @@ CONFIG_LLEXT_STORAGE_WRITABLE=y
 CONFIG_MODULES=y
 CONFIG_TIMING_FUNCTIONS=y
 CONFIG_WATCHDOG=y
+CONFIG_TIMESLICE_PER_THREAD=y
+CONFIG_THREAD_RUNTIME_STATS=y
+CONFIG_SCHED_THREAD_USAGE=y
 
 # Zephyr / device drivers
 CONFIG_CLOCK_CONTROL=y

src/Kconfig

@@ -15,6 +15,7 @@ rsource "probe/Kconfig"
 rsource "samples/Kconfig"
 
 rsource "schedule/Kconfig"
+rsource "schedule/Kconfig.threads_prio"
 
 rsource "ipc/Kconfig"
 

src/idc/zephyr_idc.c

@@ -133,7 +133,7 @@ int idc_send_msg(struct idc_msg *msg, uint32_t mode)
 	idc_send_memcpy_err = memcpy_s(msg_cp, sizeof(*msg_cp), msg, sizeof(*msg));
 	assert(!idc_send_memcpy_err);
 	/* Same priority as the IPC thread which is an EDF task and under Zephyr */
-	work->priority = EDF_ZEPHYR_PRIORITY;
+	work->priority = CONFIG_EDF_THREAD_PRIORITY;
 	work->deadline = 0;
 	work->handler = idc_handler;
 	work->sync = mode == IDC_BLOCKING;

src/include/sof/ipc/common.h

@@ -68,7 +68,11 @@ struct ipc {
 	struct list_item comp_list;	/* list of component devices */
 
 	/* processing task */
+#if CONFIG_TWB_IPC_TASK
+	struct task *ipc_task;
+#else
 	struct task ipc_task;
+#endif
 
 #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS
 	/* io performance measurement */

src/include/sof/schedule/edf_schedule.h

@@ -13,8 +13,6 @@
 #include <user/trace.h>
 #include <stdint.h>
 
-#define EDF_ZEPHYR_PRIORITY 1
-
 #define edf_sch_set_pdata(task, data) \
 	(task->priv_data = data)
 

src/include/sof/schedule/schedule.h

@@ -38,6 +38,10 @@ enum {
 				  *  and will be unified with SOF_SCHEDULE_EDF for Zephyr builds
 				  *  current implementation of Zephyr based EDF is depreciated now
 				  */
+	SOF_SCHEDULE_TWB,	/**< Tasks With Budget scheduler based on Zephyr peemptive threads
+				  * for each SOF task that has pre-allocated MCPS budget
+				  * renewed with every system tick.
+				  */
 	SOF_SCHEDULE_COUNT	/**< indicates number of scheduler types */
 };
 

src/include/sof/schedule/twb_schedule.h

@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/*
+ * Copyright(c) 2023 Intel Corporation. All rights reserved.
+ *
+ * Author: Adrian Bonislawski <[email protected]>
+ */
+
+#ifndef __SOF_SCHEDULE_TWB_SCHEDULE_H__
+#define __SOF_SCHEDULE_TWB_SCHEDULE_H__
+
+#include <rtos/task.h>
+#include <stdint.h>
+
+/**
+ * @brief Task With Budget (TWB) Scheduler
+ *
+ * TWB scheduler is a scheduler that creates a separate preemptible Zephyr thread
+ * for each SOF task that has a pre-allocated MCPS budget renewed with every system tick.
+ * The TWB scheduler assigns either MEDIUM_PRIORITY or LOW_PRIORITY to the task thread
+ * based on the budget left in the current system tick.
+ * It allows for opportunistic execution if there is no other ready task
+ * with a higher priority while the budget is already spent.
+ *
+ * Examples of tasks with budget include IPC Task and IDC Task.
+ *
+ * The TWB scheduler has two key parameters assigned:
+ * - cycles granted: the budget per system tick
+ * - cycles consumed: the number of cycles consumed in a given system tick for task execution
+ *
+ * The number of cycles consumed is reset to 0 at the beginning of each system tick,
+ * renewing the TWB budget.
+ * When the number of cycles consumed exceeds the cycles granted,
+ * the task is switchedfrom MEDIUM to LOW priority.
+ * When the task with budget thread is created, the MPP Scheduling is responsible
+ * for setting the thread time slice equal to the task budget, along with
+ * setting a callback on time slice timeout.
+ * Thread time slicing guarantees that the Zephyr scheduler will interrupt execution
+ * when the budget is spent,
+ * so the MPP Scheduling timeout callback can re-evaluate the task priority.
+ *
+ * If there is a budget left in some system tick
+ * (i.e., the task spent less time or started executing closeto the system tick
+ * that preempts execution), it is reset and not carried over to the next tick.
+ *
+ * More info:
+ * https://thesofproject.github.io/latest/architectures/firmware/sof-zephyr/mpp_layer/mpp_scheduling.html
+ */
+
+/**
+ * \brief default static stack size for each TWB thread
+ */
+#define ZEPHYR_TWB_STACK_SIZE 8192
+
+/**
+ * \brief max budget limit
+ */
+#define ZEPHYR_TWB_BUDGET_MAX (CONFIG_SYS_CLOCK_TICKS_PER_SEC / 1000)
+
+#define SYS_TICKS_TO_HW_CYCLES(x) (x * CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC / CONFIG_SYS_CLOCK_TICKS_PER_SEC)
+#define HW_CYCLES_TO_SYS_TICKS(x) (x * CONFIG_SYS_CLOCK_TICKS_PER_SEC / CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC)
+
+/**
+ * \brief Init the Tasks with Budget scheduler
+ */
+int scheduler_twb_init(void);
+
+/**
+ * \brief initialize a TWB task and add it to scheduling
+ * It must be called on core the task is declared to run on
+ *
+ * \param[out] task pointer, pointer to allocated task structure will be return
+ * \param[in] uid pointer to UUID of the task
+ * \param[in] ops pointer to task functions
+ * \param[in] data pointer to the task data
+ * \param[in] core CPU the thread should run on
+ * \param[in] name zephyr thread name
+ * \param[in] stack_size size of stack for a zephyr thread
+ * \param[in] thread_priority priority of the zephyr thread
+ * \param[in] cycles_granted cycles budget for the zephyr thread
+ */
+int scheduler_twb_task_init(struct task **task,
+			    const struct sof_uuid_entry *uid,
+			    const struct task_ops *ops,
+			    void *data,
+			    int32_t core,
+			    const char *name,
+			    size_t stack_size,
+			    int32_t thread_priority,
+			    uint32_t cycles_granted);
+
+#endif /* __SOF_SCHEDULE_TWB_SCHEDULE_H__ */

src/ipc/ipc-common.c

@@ -273,7 +273,11 @@ static void ipc_work_handler(struct k_work *work)
 
 void ipc_schedule_process(struct ipc *ipc)
 {
+#if CONFIG_TWB_IPC_TASK
+	schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC);
+#else
 	schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC);
+#endif
 }
 
 int ipc_init(struct sof *sof)

src/ipc/ipc-zephyr.c

@@ -31,6 +31,7 @@
 #include <sof/list.h>
 #include <sof/platform.h>
 #include <sof/schedule/edf_schedule.h>
+#include <sof/schedule/twb_schedule.h>
 #include <sof/schedule/schedule.h>
 #include <rtos/task.h>
 #include <rtos/spinlock.h>
@@ -160,9 +161,14 @@ static int ipc_device_resume_handler(const struct device *dev, void *arg)
 	intel_adsp_ipc_set_message_handler(INTEL_ADSP_IPC_HOST_DEV, message_handler, ipc);
 
 	/* schedule task */
+#if CONFIG_TWB_IPC_TASK
+	scheduler_twb_task_init(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
+							&ipc_task_ops, ipc, 0, "IPC", ZEPHYR_TWB_STACK_SIZE,
+							CONFIG_TWB_THREAD_MEDIUM_PRIORITY, ZEPHYR_TWB_BUDGET_MAX / 2);
+#else
 	schedule_task_init_edf(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
 			       &ipc_task_ops, ipc, 0, 0);
-
+#endif
 	return 0;
 }
 #endif /* CONFIG_PM_DEVICE */
@@ -278,9 +284,14 @@ int platform_ipc_init(struct ipc *ipc)
 	ipc_set_drvdata(ipc, NULL);
 
 	/* schedule task */
+#if CONFIG_TWB_IPC_TASK
+	scheduler_twb_task_init(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
+							&ipc_task_ops, ipc, 0, "IPC", ZEPHYR_TWB_STACK_SIZE,
+							CONFIG_TWB_THREAD_MEDIUM_PRIORITY, ZEPHYR_TWB_BUDGET_MAX / 2);
+#else
 	schedule_task_init_edf(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
 			       &ipc_task_ops, ipc, 0, 0);
-
+#endif
 	/* configure interrupt - work is done internally by Zephyr API */
 
 	/* attach handlers */

src/platform/intel/ace/platform.c

@@ -18,6 +18,7 @@
 #include <sof/lib/cpu-clk-manager.h>
 #include <sof/schedule/edf_schedule.h>
 #include <sof/schedule/dp_schedule.h>
+#include <sof/schedule/twb_schedule.h>
 #include <sof/schedule/ll_schedule.h>
 #include <sof/schedule/ll_schedule_domain.h>
 #include <sof/trace/trace.h>
@@ -115,6 +116,12 @@ int platform_init(struct sof *sof)
 		return ret;
 #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */
 
+#if CONFIG_ZEPHYR_TWB_SCHEDULER
+	ret = scheduler_twb_init();
+	if (ret < 0)
+		return ret;
+#endif
+
 	/* init the system agent */
 	trace_point(TRACE_BOOT_PLATFORM_AGENT);
 	sa_init(sof, CONFIG_SYSTICK_PERIOD);

src/schedule/Kconfig

@@ -51,3 +51,22 @@ config SCHEDULE_LL_NO_RESCHEDULE_TASK
 	  scheduler_ops::reschedule_task will set to NULL instead, tasks with
 	  the attempt to reschedule (e.g. DMA trace works) will be relinguished
 	  directly and return no error.
+
+config ZEPHYR_TWB_SCHEDULER
+	bool "use Zephyr thread based TWB scheduler"
+	default n
+	depends on ZEPHYR_SOF_MODULE
+	depends on TIMESLICE_PER_THREAD
+	depends on THREAD_RUNTIME_STATS
+	depends on SCHED_THREAD_USAGE
+	help
+	  Enable Tasks with Budget preemptive scheduler based on
+	  Zephyr preemptive threads for each SOF task that has pre-allocated
+	  MCPS budget renewed with every system tick.
+
+config TWB_IPC_TASK
+	bool "use TWB scheduler for IPC task"
+	default n
+	depends on ZEPHYR_TWB_SCHEDULER
+	help
+	  Switch IPC task to TWB scheduler.

src/schedule/Kconfig.threads_prio

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+config LL_THREAD_PRIORITY
+	int "LL thread cooperative high priority"
+	default -16
+	help
+	  LL thread configured priority in the system.
+	  Should be in cooperative range:
+	  -NUM_COOP_PRIORITIES to -1
+
+config TWB_THREAD_MEDIUM_PRIORITY
+	int "TWB thread preemptible medium priority"
+	default 1
+	help
+	  TWB thread configured priority in the system.
+	  Should be in preemptible range:
+	  0 to NUM_PREEMPT_PRIORITIES-1
+
+config TWB_THREAD_LOW_PRIORITY
+	int "TWB thread preemptible low priority"
+	default 12
+	help
+	  TWB thread configured priority in the system.
+	  Should be in preemptible range:
+	  0 to NUM_PREEMPT_PRIORITIES-1
+
+config DP_THREAD_PRIORITY
+	int "DP thread preemptible low priority"
+	default 12
+	help
+	  DP thread configured priority in the system.
+	  Should be in preemptible range:
+	  0 to NUM_PREEMPT_PRIORITIES-1
+
+config EDF_THREAD_PRIORITY
+	int "EDF thread preemptible low priority"
+	default 1
+	help
+	  EDF thread configured priority in the system.
+	  Should be in preemptible range:
+	  0 to NUM_PREEMPT_PRIORITIES-1

src/schedule/zephyr_dma_domain.c

@@ -450,7 +450,7 @@ static int zephyr_dma_domain_register(struct ll_schedule_domain *domain,
 				 dt,
 				 NULL,
 				 NULL,
-				 -CONFIG_NUM_COOP_PRIORITIES,
+				 CONFIG_LL_THREAD_PRIORITY,
 				 0,
 				 K_FOREVER);
 

src/schedule/zephyr_domain.c

@@ -195,7 +195,7 @@ static int zephyr_domain_register(struct ll_schedule_domain *domain,
 				 ll_sched_stack[core],
 				 ZEPHYR_LL_STACK_SIZE,
 				 zephyr_domain_thread_fn, zephyr_domain, NULL, NULL,
-				 -CONFIG_NUM_COOP_PRIORITIES, 0, K_FOREVER);
+				 CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER);
 
 	k_thread_cpu_mask_clear(thread);
 	k_thread_cpu_mask_enable(thread, core);

src/schedule/zephyr_dp_schedule.c

@@ -27,11 +27,6 @@ SOF_DEFINE_REG_UUID(dp_sched);
 
 DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO);
 
-/**
- * \brief a priority of the DP threads in the system.
- */
-#define ZEPHYR_DP_THREAD_PRIORITY (CONFIG_NUM_PREEMPT_PRIORITIES - 2)
-
 struct scheduler_dp_data {
 	struct list_item tasks;		/* list of active dp tasks */
 	struct task ll_tick_src;	/* LL task - source of DP tick */
@@ -399,7 +394,7 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta
 	/* create a zephyr thread for the task */
 	pdata->thread_id = k_thread_create(&pdata->thread, (__sparse_force void *)pdata->p_stack,
 					   pdata->stack_size, dp_thread_fn, task, NULL, NULL,
-					   ZEPHYR_DP_THREAD_PRIORITY, K_USER, K_FOREVER);
+					   CONFIG_DP_THREAD_PRIORITY, K_USER, K_FOREVER);
 
 	/* pin the thread to specific core */
 	ret = k_thread_cpu_pin(pdata->thread_id, task->core);