Support turbo mode for better latency but burn more power on CPU & NPU (

#250)

Signed-off-by: Min Ma <min.ma@amd.com>

src/driver/amdxdna/aie2_message.c

@@ -249,6 +249,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 {
 	DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
 	struct amdxdna_dev *xdna = ndev->xdna;
+	enum xdna_mailbox_channel_type type;
 	struct xdna_mailbox_chann_res x2i;
 	struct xdna_mailbox_chann_res i2x;
 	struct cq_pair *cq_pair;
@@ -287,8 +288,12 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 	}
 
 	intr_reg = i2x.mb_head_ptr_reg + 4;
+	if (aie2_pm_is_turbo(ndev))
+		type = MB_CHANNEL_USER_POLL;
+	else
+		type = MB_CHANNEL_USER_NORMAL;
 	hwctx->priv->mbox_chann = xdna_mailbox_create_channel(ndev->mbox, &x2i, &i2x,
-							      intr_reg, ret);
+							      intr_reg, ret, type);
 	if (!hwctx->priv->mbox_chann) {
 		XDNA_ERR(xdna, "not able to create channel");
 		ret = -EINVAL;

src/driver/amdxdna/aie2_msg_priv.h

@@ -186,7 +186,6 @@ struct exec_dpu_req {
 	u32     inst_prop_cnt;
 	u32     cu_idx;
 	u32	payload[35];
-
 } __packed;
 
 struct exec_dpu_preempt_req {

src/driver/amdxdna/aie2_pci.c

@@ -428,7 +428,7 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
 						       &ndev->mgmt_x2i,
 						       &ndev->mgmt_i2x,
 						       xdna_mailbox_intr_reg,
-						       mgmt_mb_irq);
+						       mgmt_mb_irq, MB_CHANNEL_MGMT);
 	if (!ndev->mgmt_chann) {
 		XDNA_ERR(xdna, "failed to create management mailbox channel");
 		ret = -EINVAL;
@@ -576,6 +576,7 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	aie2_smu_setup(ndev);
 
 	ndev->pw_mode = POWER_MODE_DEFAULT;
+	ndev->clk_gate_enabled = true;
 	ret = aie2_hw_start(xdna);
 	if (ret) {
 		XDNA_ERR(xdna, "start npu failed, ret %d", ret);
@@ -986,9 +987,8 @@ static int aie2_set_power_mode(struct amdxdna_client *client, struct amdxdna_drm
 		return -EFAULT;
 	}
 
-	/* Interpret the given buf->power_mode into the correct power mode*/
 	power_mode = power_state.power_mode;
-	if (power_mode > POWER_MODE_HIGH) {
+	if (power_mode > POWER_MODE_TURBO) {
 		XDNA_ERR(xdna, "Invalid power mode %d", power_mode);
 		return -EINVAL;
 	}

src/driver/amdxdna/aie2_pci.h

@@ -223,6 +223,7 @@ struct amdxdna_dev_hdl {
 	struct aie_metadata		metadata;
 	struct smu			smu;
 	enum amdxdna_power_mode_type	pw_mode;
+	bool				clk_gate_enabled;
 
 	/* Mailbox and the management channel */
 	struct mailbox			*mbox;
@@ -368,6 +369,7 @@ void aie2_stop_ctx_by_col_map(struct amdxdna_client *client, u32 col_map);
 /* aie2_pm.c */
 int aie2_pm_start(struct amdxdna_dev_hdl *ndev);
 void aie2_pm_stop(struct amdxdna_dev_hdl *ndev);
+bool aie2_pm_is_turbo(struct amdxdna_dev_hdl *ndev);
 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target);
 
 #endif /* _AIE2_PCI_H_ */

src/driver/amdxdna/aie2_pm.c

@@ -5,13 +5,19 @@
 
 #include "aie2_pci.h"
 
-static int aie2_pm_clock_gating(struct amdxdna_dev_hdl *ndev, bool enable)
+static int aie2_pm_clock_gating(struct amdxdna_dev_hdl *ndev,
+				enum amdxdna_power_mode_type target)
 {
 	const struct rt_config_clk_gating *config;
+	bool enable;
 	u32 value;
 	int ret;
 	int i;
 
+	enable = (target != POWER_MODE_TURBO && target != POWER_MODE_HIGH);
+	if (enable == ndev->clk_gate_enabled)
+		return 0;
+
 	config = &ndev->priv->clk_gating;
 	if (enable)
 		value = config->value_enable;
@@ -30,9 +36,40 @@ static int aie2_pm_clock_gating(struct amdxdna_dev_hdl *ndev, bool enable)
 		}
 	}
 
+	if (!ret)
+		ndev->clk_gate_enabled = enable;
+
 	return ret;
 }
 
+bool aie2_pm_is_turbo(struct amdxdna_dev_hdl *ndev)
+{
+	return ndev->pw_mode == POWER_MODE_TURBO;
+}
+
+static int aie2_pm_check_turbo(struct amdxdna_dev_hdl *ndev,
+			       enum amdxdna_power_mode_type prev,
+			       enum amdxdna_power_mode_type next)
+{
+	struct amdxdna_dev *xdna = ndev->xdna;
+	struct amdxdna_client *client;
+
+	if (prev != POWER_MODE_TURBO && next != POWER_MODE_TURBO)
+		return 0;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+	list_for_each_entry(client, &xdna->client_list, node) {
+		bool empty;
+
+		mutex_lock(&client->hwctx_lock);
+		empty = idr_is_empty(&client->hwctx_idr);
+		mutex_unlock(&client->hwctx_lock);
+		if (!empty)
+			return -EBUSY;
+	}
+	return 0;
+}
+
 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target)
 {
 	struct amdxdna_dev *xdna = ndev->xdna;
@@ -44,23 +81,29 @@ int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type
 	if (target == POWER_MODE_LOW || target == POWER_MODE_MEDIUM)
 		return -EOPNOTSUPP;
 
-	XDNA_DBG(xdna, "Changing power mode from %d to %d", ndev->pw_mode, target);
-	/* Set resource solver power property to the user choice */
+	ret = aie2_pm_check_turbo(ndev, ndev->pw_mode, target);
+	if (ret) {
+		XDNA_WARN(xdna, "Change Turbo mode failed");
+		return ret;
+	}
 
-	/* Set power level within the device */
+	XDNA_DBG(xdna, "Changing power mode from %d to %d", ndev->pw_mode, target);
 
-	/*
-	 * Other mode -> POWER_MODE_HIGH: Turn off clock gating
-	 * POWER_MODE_HIGH -> Other mode: Turn on clock gating
-	 * Otherwise, no change
+	/* TODO:
+	 *switch (ndev->pw_mode) {
+	 *case POWER_MODE_LOW:
+	 *	Set to low DPM level
+	 *case POWER_MODE_MEDIUM:
+	 *	Set to medium DPM level
+	 *case POWER_MODE_HIGH:
+	 *case POWER_MODE_TURBO:
+	 *	Set to highest DPM level
+	 *default:
+	 *	Let driver decides DPM level
+	 *}
 	 */
-	if (target == POWER_MODE_HIGH) {
-		XDNA_DBG(xdna, "Clock gating turning off");
-		ret = aie2_pm_clock_gating(ndev, false);
-	} else if (ndev->pw_mode == POWER_MODE_HIGH) {
-		XDNA_DBG(xdna, "Clock gating turning on");
-		ret = aie2_pm_clock_gating(ndev, true);
-	}
+
+	ret = aie2_pm_clock_gating(ndev, target);
 	if (ret) {
 		XDNA_ERR(xdna, "Failed to config clock gating");
 		return ret;
@@ -73,21 +116,10 @@ int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type
 
 int aie2_pm_start(struct amdxdna_dev_hdl *ndev)
 {
-	/*
-	 * TODO: should only skip POWER_MODE_DEFAULT.
-	 * Let's make it right after full DPM support is ready
-	 */
-	if (ndev->pw_mode != POWER_MODE_HIGH)
-		return 0;
-
-	return aie2_pm_clock_gating(ndev, false);
+	return aie2_pm_clock_gating(ndev, ndev->pw_mode);
 }
 
 void aie2_pm_stop(struct amdxdna_dev_hdl *ndev)
 {
-	if (ndev->pw_mode != POWER_MODE_HIGH)
-		return;
-
-	/* Clock gating must be turned ON before suspend firmware */
-	aie2_pm_clock_gating(ndev, true);
+	aie2_pm_clock_gating(ndev, POWER_MODE_DEFAULT);
 }