OpenSBI的HSM

HSM规范

HSM是指硬件线程状态管理（Hart State Management），其引入了一组Hart状态和一些函数，这些函数允许S模式软件请求改变Hart状态。下表定义了一些HSM Hart状态：

在任何时刻，hart 必须处于上述状态之一，SBI 实现的 hart 状态转换应遵循下图所示的状态机。

HSM函数

SBI规范中定义了相关HSM函数，用于获取和改变 hart 的状态，如下表所示：

Hart启动（FID#0）

struct sbiret sbi_hart_start(unsigned long hartid,
                             unsigned long start_addr,
                             unsigned long opaque)

请求 SBI 启动Hart运行到S模式，参数说明如下：

• hartid：启动哪个hart
• start_addr：运行启动地址
• opaque：值为a1寄存器（arg1）

该函数执行可能返回如下结果：

Hart 停止 （FID#1）

struct sbiret sbi_hart_stop(void)

请求 SBI 停止执行S模式 hart，并将其所有权返回给 SBI。该函数必须在中断关闭的情况下调用，可能返回的错误码如下：

Hart 获取状态（FID#2）

struct sbiret sbi_hart_get_status(unsigned long hartid)

获取给定 hart 的当前状态（或 HSM 状态 ID），可能返回错误：

由于存在并发调用sbi_hart_start()、sbi_hart_stop() 或 sbi_hart_suspend()，harts 可能随时转换 HSM 状态，因此该函数的返回值可能并不代表 hart 的实际状态。

HART 挂起 （FID#3）

struct sbiret sbi_hart_suspend(uint32_t suspend_type,
                               unsigned long resume_addr,
                               unsigned long opaque)

请求 SBI 将 hart 变为挂起（或低功耗）状态。当收到中断或特定的硬件事件时，hart 将自动退出挂起状态并恢复正常执行。

• suspend_type：挂起类型

  值	描述
  0x00000000	默认保持性挂起
  0x00000001 - 0x0FFFFFFF	保留
  0x10000000 - 0x7FFFFFFF	特定平台保持性挂起
  0x80000000	默认非保持性挂起
  0x80000001 - 0x8FFFFFFF	保留
  0x90000000 - 0xFFFFFFFF	特定平台非保持性挂起

• resume_addr：在一个非保持性挂起后，hart 恢复执行的物理地址

• opaque：a1寄存器

该函数可能返回的错误如下：

代码

HSM初始化

在OpenSBI的启动代码 lib/sbi/sbi_hsm.c 中，进行了HSM的初始化：

int sbi_hsm_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot)
{
	u32 i;
	struct sbi_scratch *rscratch;
	struct sbi_hsm_data *hdata;

	if (cold_boot) {
		hart_data_offset = sbi_scratch_alloc_offset(sizeof(*hdata));
		if (!hart_data_offset)
			return SBI_ENOMEM;

		/* Initialize hart state data for every hart */
		for (i = 0; i <= sbi_scratch_last_hartindex(); i++) {
			rscratch = sbi_hartindex_to_scratch(i);
			if (!rscratch)
				continue;

			hdata = sbi_scratch_offset_ptr(rscratch,
						       hart_data_offset);
			ATOMIC_INIT(&hdata->state,
				    (sbi_hartindex_to_hartid(i) == hartid) ?
				    SBI_HSM_STATE_START_PENDING :
				    SBI_HSM_STATE_STOPPED);
			ATOMIC_INIT(&hdata->start_ticket, 0);
		}
	} else {
		sbi_hsm_hart_wait(scratch, hartid);
	}

	return 0;
}

如果为冷启动，分配一个struct sbi_hsm_data类型的结构体，并设置 Hart 的初始状态：如果是启动Hart，设置为等待启动状态SBI_HSM_STATE_START_PENDING，否则设置为结束状态SBI_HSM_STATE_STOPPED。

/** Per hart specific data to manage state transition **/
struct sbi_hsm_data {
	atomic_t state;
	unsigned long suspend_type;
	unsigned long saved_mie;
	unsigned long saved_mip;
	unsigned long saved_medeleg;
	unsigned long saved_menvcfg;
#if __riscv_xlen == 32
	unsigned long saved_menvcfgh;
#endif
	atomic_t start_ticket;
};

如果为其他类型启动，调用sbi_hsm_hart_wait函数，首先保存MIE中断使能寄存器，配置MIE使能M模式的软件中断和外部中断，然后调用wif()进入低功耗状态，等待被唤醒，即当该Hart的状态变为等待被启动SBI_HSM_STATE_START_PENDING时，就会退出wfi状态，并恢复MIE中断使能寄存器：

static void sbi_hsm_hart_wait(struct sbi_scratch *scratch, u32 hartid)
{
	unsigned long saved_mie;
	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
							    hart_data_offset);
	/* Save MIE CSR */
	saved_mie = csr_read(CSR_MIE);

	/* Set MSIE and MEIE bits to receive IPI */
	csr_set(CSR_MIE, MIP_MSIP | MIP_MEIP);

	/* Wait for state transition requested by sbi_hsm_hart_start() */
	while (atomic_read(&hdata->state) != SBI_HSM_STATE_START_PENDING) {
		wfi();
	}

	/* Restore MIE CSR */
	csr_write(CSR_MIE, saved_mie);

	/*
	 * No need to clear IPI here because the sbi_ipi_init() will
	 * clear it for current HART via sbi_platform_ipi_init().
	 */
}

HSM的ECALL

在文件lib/sbi/sbi_ecall_hsm.c的sbi_ecall_hsm_handler函数中，实现了对HSM相关ECALL的处理，根据传入参数funcid进行不同的HSM函数的处理：

static int sbi_ecall_hsm_handler(unsigned long extid, unsigned long funcid,
				 struct sbi_trap_regs *regs,
				 struct sbi_ecall_return *out)
{
	int ret = 0;
	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
	ulong smode = (csr_read(CSR_MSTATUS) & MSTATUS_MPP) >>
			MSTATUS_MPP_SHIFT;

	switch (funcid) {
	case SBI_EXT_HSM_HART_START:
		ret = sbi_hsm_hart_start(scratch, sbi_domain_thishart_ptr(),
					 regs->a0, regs->a1, smode, regs->a2);
		break;
	case SBI_EXT_HSM_HART_STOP:
		ret = sbi_hsm_hart_stop(scratch, true);
		break;
	case SBI_EXT_HSM_HART_GET_STATUS:
		ret = sbi_hsm_hart_get_state(sbi_domain_thishart_ptr(),
					     regs->a0);
		break;
	case SBI_EXT_HSM_HART_SUSPEND:
		ret = sbi_hsm_hart_suspend(scratch, regs->a0, regs->a1,
					   smode, regs->a2);
		break;
	default:
		ret = SBI_ENOTSUPP;
	}

	if (ret >= 0) {
		out->value = ret;
		ret = 0;
	}

	return ret;
}

sbi_hsm_hart_start

函数sbi_hsm_hart_start是启动Hart，该请求来自另外一个Hart，实现如下：

int sbi_hsm_hart_start(struct sbi_scratch *scratch,
		       const struct sbi_domain *dom,
		       u32 hartid, ulong saddr, ulong smode, ulong arg1)
{
	unsigned long init_count, entry_count;
	unsigned int hstate;
	struct sbi_scratch *rscratch;
	struct sbi_hsm_data *hdata;
	int rc;

	/* For now, we only allow start mode to be S-mode or U-mode. */
	if (smode != PRV_S && smode != PRV_U)
		return SBI_EINVAL;
	if (dom && !sbi_domain_is_assigned_hart(dom, hartid))
		return SBI_EINVAL;
	if (dom && !sbi_domain_check_addr(dom, saddr, smode,
					  SBI_DOMAIN_EXECUTE))
		return SBI_EINVALID_ADDR;

	rscratch = sbi_hartid_to_scratch(hartid);
	if (!rscratch)
		return SBI_EINVAL;

	hdata = sbi_scratch_offset_ptr(rscratch, hart_data_offset);
	if (!hsm_start_ticket_acquire(hdata))
		return SBI_EINVAL;

	init_count = sbi_init_count(hartid);
	entry_count = sbi_entry_count(hartid);

	rscratch->next_arg1 = arg1;
	rscratch->next_addr = saddr;
	rscratch->next_mode = smode;

	/*
	 * atomic_cmpxchg() is an implicit barrier. It makes sure that
	 * other harts see reading of init_count and writing to *rscratch
	 * before hdata->state is set to SBI_HSM_STATE_START_PENDING.
	 */
	hstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_STOPPED,
				SBI_HSM_STATE_START_PENDING);
	if (hstate == SBI_HSM_STATE_STARTED) {
		rc = SBI_EALREADY;
		goto err;
	}

	/**
	 * if a hart is already transition to start or stop, another start call
	 * is considered as invalid request.
	 */
	if (hstate != SBI_HSM_STATE_STOPPED) {
		rc = SBI_EINVAL;
		goto err;
	}

	if ((hsm_device_has_hart_hotplug() && (entry_count == init_count)) ||
	   (hsm_device_has_hart_secondary_boot() && !init_count)) {
		rc = hsm_device_hart_start(hartid, scratch->warmboot_addr);
	} else {
		rc = sbi_ipi_raw_send(sbi_hartid_to_hartindex(hartid));
	}

	if (!rc)
		return 0;

	/* If it fails to start, change hart state back to stop */
	__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_START_PENDING,
				    SBI_HSM_STATE_STOPPED);
err:
	hsm_start_ticket_release(hdata);
	return rc;
}

• 首先进行参数检查：判断要启动的是否为S模式或者U模式，Hart是否分配到该Domain，要启动的地址是否允许执行；
• 接着获取该Hart的初始化次数和进入次数；
• 然后调用atomic_cmpxchg进行状态切换，先判断Hart是否处于是否处于结束状态SBI_HSM_STATE_STOPPED（上一节HSM初始化会将该Hart置于结束状态），如果是就将该Hart切换到等待启动状态SBI_HSM_STATE_START_PENDING，并返回Hart之前的状态hstate，如果之前该Hart状态为SBI_HSM_STATE_STARTED，说明之前已经启动过；如果之前该Hart状态不是SBI_HSM_STATE_STOPPED，说明是非法请求，返回错误；
• 最后判断设备是否支持 hart 的热拔插，或者支持二级启动；如果是，则调用 hsm_device_hart_start 设备函数来实现状态切换，否则发送IPI中断。

当上面待启动的Hart收到中断事件唤醒后，发现自己处于SBI_HSM_STATE_START_PENDING，就会退出wfi()：

/* Wait for state transition requested by sbi_hsm_hart_start() */
while (atomic_read(&hdata->state) != SBI_HSM_STATE_START_PENDING) {
    wfi();
}

sbi_hsm_hart_stop

函数sbi_hsm_hart_stop是停止执行该Hart，实现如下：

int sbi_hsm_hart_stop(struct sbi_scratch *scratch, bool exitnow)
{
	const struct sbi_domain *dom = sbi_domain_thishart_ptr();
	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
							    hart_data_offset);

	if (!dom)
		return SBI_EFAIL;

	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_STARTED,
					 SBI_HSM_STATE_STOP_PENDING))
		return SBI_EFAIL;

	if (exitnow)
		sbi_exit(scratch);

	return 0;
}

sbi_hsm_hart_get_state

函数sbi_hsm_hart_get_state是获取该Hart的状态，实现如下：

int __sbi_hsm_hart_get_state(u32 hartid)
{
	struct sbi_hsm_data *hdata;
	struct sbi_scratch *scratch;

	scratch = sbi_hartid_to_scratch(hartid);
	if (!scratch)
		return SBI_EINVAL;

	hdata = sbi_scratch_offset_ptr(scratch, hart_data_offset);
	return atomic_read(&hdata->state);
}

int sbi_hsm_hart_get_state(const struct sbi_domain *dom, u32 hartid)
{
	if (!sbi_domain_is_assigned_hart(dom, hartid))
		return SBI_EINVAL;

	return __sbi_hsm_hart_get_state(hartid);
}

sbi_hsm_hart_suspend

函数sbi_hsm_hart_suspend是请求将该Hart变为挂起状态，实现如下：

int sbi_hsm_hart_suspend(struct sbi_scratch *scratch, u32 suspend_type,
			 ulong raddr, ulong rmode, ulong arg1)
{
	int ret;
	const struct sbi_domain *dom = sbi_domain_thishart_ptr();
	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
							    hart_data_offset);

	/* Sanity check on domain assigned to current HART */
	if (!dom)
		return SBI_EFAIL;

	/* Sanity check on suspend type */
	if (SBI_HSM_SUSPEND_RET_DEFAULT < suspend_type &&
	    suspend_type < SBI_HSM_SUSPEND_RET_PLATFORM)
		return SBI_EINVAL;
	if (SBI_HSM_SUSPEND_NON_RET_DEFAULT < suspend_type &&
	    suspend_type < SBI_HSM_SUSPEND_NON_RET_PLATFORM)
		return SBI_EINVAL;

	/* Additional sanity check for non-retentive suspend */
	if (suspend_type & SBI_HSM_SUSP_NON_RET_BIT) {
		/*
		 * For now, we only allow non-retentive suspend from
		 * S-mode or U-mode.
		 */
		if (rmode != PRV_S && rmode != PRV_U)
			return SBI_EFAIL;
		if (dom && !sbi_domain_check_addr(dom, raddr, rmode,
						  SBI_DOMAIN_EXECUTE))
			return SBI_EINVALID_ADDR;
	}

	/* Save the resume address and resume mode */
	scratch->next_arg1 = arg1;
	scratch->next_addr = raddr;
	scratch->next_mode = rmode;

	/* Directly move from STARTED to SUSPENDED state */
	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_STARTED,
					 SBI_HSM_STATE_SUSPENDED))
		return SBI_EFAIL;

	/* Save the suspend type */
	hdata->suspend_type = suspend_type;

	/*
	 * Save context which will be restored after resuming from
	 * non-retentive suspend.
	 */
	if (suspend_type & SBI_HSM_SUSP_NON_RET_BIT)
		__sbi_hsm_suspend_non_ret_save(scratch);

	/* Try platform specific suspend */
	ret = hsm_device_hart_suspend(suspend_type);
	if (ret == SBI_ENOTSUPP) {
		/* Try generic implementation of default suspend types */
		if (suspend_type == SBI_HSM_SUSPEND_RET_DEFAULT ||
		    suspend_type == SBI_HSM_SUSPEND_NON_RET_DEFAULT) {
			ret = __sbi_hsm_suspend_default(scratch);
		}
	}

	/*
	 * The platform may have coordinated a retentive suspend, or it may
	 * have exited early from a non-retentive suspend. Either way, the
	 * caller is not expecting a successful return, so jump to the warm
	 * boot entry point to simulate resume from a non-retentive suspend.
	 */
	if (ret == 0 && (suspend_type & SBI_HSM_SUSP_NON_RET_BIT)) {
		void (*jump_warmboot)(void) =
			(void (*)(void))scratch->warmboot_addr;

		jump_warmboot();
	}

	/*
	 * We might have successfully resumed from retentive suspend
	 * or suspend failed. In both cases, we restore state of hart.
	 */
	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_SUSPENDED,
					 SBI_HSM_STATE_STARTED))
		sbi_hart_hang();

	return ret;
}

• 首先检查挂起类型，目前要么为默认保持性挂起，要么为默认非保持性挂起，其他返回错误。对于非保持性挂起，还要检查是否为S模式或者U模式以及恢复地址是否允许执行；
• 接着保存恢复地址和模式，并将该Hart从启动状态SBI_HSM_STATE_STARTED切换到挂起状态SBI_HSM_STATE_SUSPENDED，并保存挂起类型，另外对于非保持性挂起，还需要进行Hart上下文保存；
• 然后执行设备挂起函数hsm_device_hart_suspend，如果返回不支持，就尝试执行默认的挂起类型，Hart直接进入WFI；
• 最后当Hart从非保持性挂起状态中恢复时，就跳转到热启动地址jump_warmboot；而对于保持性挂起状态中恢复后，进行Hart状态切换，即从等待挂起状态SBI_HSM_STATE_SUSPENDED切换到等待启动状态SBI_HSM_STATE_STARTED。

参考

1. Hart State Management Extension
2. RISC-V SBI 翻译
3. OpenSBI 固件分析与 SBI 规范的 HSM 扩展