[VSC-1546] fix version switch by persist sys python in idfSetup (#1384)

* fix version switch by persist sys python in idfSetup

* idf pythonInstallPath saved as workspace folder setting

* fix project tests

* log cCppPropertiesJson

* add more logs

* fix lint

* update rw c cpp properties json

* test ci python install path

* add both venv sys python

* only return idf-python if path exists

* fallback save pythonInstallPath

* getVirtualEnvPythonPath only existing path

* fix lint

* getCurrentIdfSetup use pythonBinPath if available

* update example creation to use idfSetup python if exist

* useIdfSetupSettings use idfSetup python if exist

package.json

@@ -678,7 +678,7 @@
             "type": "string",
             "default": "python",
             "description": "%param.pythonBinPath%",
-            "scope": "application"
+            "scope": "resource"
           },
           "idf.flashBaudRate": {
             "type": "string",

src/checkExtensionSettings.ts

@@ -80,6 +80,7 @@ export async function checkExtensionSettings(
       idfPath,
       gitPath,
       toolsPath: idfToolsPath,
+      sysPythonPath: "/usr/bin/python3",
       version: idfVersion,
       isValid: false,
     };

src/examples/ExamplesPanel.ts

@@ -23,6 +23,7 @@ import { getExamplesList, IExampleCategory } from "./Example";
 import { ComponentManagerUIPanel } from "../component-manager/panel";
 import { OutputChannel } from "../logger/outputChannel";
 import { IdfSetup } from "../views/setup/types";
+import { getSystemPythonFromSettings } from "../pythonManager";
 
 export class ExamplesPlanel {
   public static currentPanel: ExamplesPlanel | undefined;
@@ -122,8 +123,7 @@ export class ExamplesPlanel {
               );
               await this.setCurrentSettingsInTemplate(
                 settingsJsonPath,
-                idfSetup.idfPath,
-                idfSetup.toolsPath
+                idfSetup
               );
               vscode.commands.executeCommand("vscode.openFolder", projectPath);
             } catch (error) {
@@ -219,13 +219,21 @@ export class ExamplesPlanel {
 
   private async setCurrentSettingsInTemplate(
     settingsJsonPath: string,
-    idfPathDir: string,
-    toolsPath: string
+    idfSetup: IdfSetup
   ) {
     const settingsJson = await readJSON(settingsJsonPath);
     const isWin = process.platform === "win32" ? "Win" : "";
-    settingsJson["idf.espIdfPath" + isWin] = idfPathDir;
-    settingsJson["idf.toolsPath" + isWin] = toolsPath;
+    settingsJson["idf.espIdfPath" + isWin] = idfSetup.idfPath;
+    settingsJson["idf.toolsPath" + isWin] = idfSetup.toolsPath;
+    if (idfSetup.python) {
+      settingsJson["idf.pythonInstallPath"] = await getSystemPythonFromSettings(
+        idfSetup.python,
+        idfSetup.idfPath,
+        idfSetup.toolsPath
+      );
+    } else {
+      settingsJson["idf.pythonInstallPath"] = idfSetup.sysPythonPath;
+    }
     await writeJSON(settingsJsonPath, settingsJson, {
       spaces: vscode.workspace.getConfiguration().get("editor.tabSize") || 2,
     });

src/newProject/newProjectInit.ts

@@ -27,8 +27,7 @@ import { getPreviousIdfSetups } from "../setup/existingIdfSetups";
 import { IdfSetup } from "../views/setup/types";
 
 export interface INewProjectArgs {
-  espIdfPath: string;
-  espIdfToolsPath: string;
+  espIdfSetup: IdfSetup;
   espAdfPath: string;
   espMdfPath: string;
   espMatterPath: string;
@@ -149,9 +148,8 @@ export async function getNewProjectArgs(
   return {
     boards: espBoards,
     components,
-    espIdfToolsPath: idfSetup.toolsPath,
+    espIdfSetup: idfSetup,
     espAdfPath: adfExists ? espAdfPath : undefined,
-    espIdfPath: idfExists ? idfSetup.idfPath : undefined,
     espMdfPath: mdfExists ? espMdfPath : undefined,
     espMatterPath: matterExists ? espMatterPath : undefined,
     espHomeKitSdkPath: homekitSdkExists ? espHomeKitSdkPath : undefined,

src/newProject/newProjectPanel.ts

@@ -22,6 +22,7 @@ import * as utils from "../utils";
 import { IExample } from "../examples/Example";
 import { setCurrentSettingsInTemplate } from "./utils";
 import { NotificationMode, readParameter } from "../idfConfiguration";
+import { IdfSetup } from "../views/setup/types";
 
 export class NewProjectPanel {
   public static currentPanel: NewProjectPanel | undefined;
@@ -70,8 +71,8 @@ export class NewProjectPanel {
     if (newProjectArgs.espAdfPath) {
       localResourceRoots.push(vscode.Uri.file(newProjectArgs.espAdfPath));
     }
-    if (newProjectArgs.espIdfPath) {
-      localResourceRoots.push(vscode.Uri.file(newProjectArgs.espIdfPath));
+    if (newProjectArgs.espIdfSetup.idfPath) {
+      localResourceRoots.push(vscode.Uri.file(newProjectArgs.espIdfSetup.idfPath));
     }
     if (newProjectArgs.espMdfPath) {
       localResourceRoots.push(vscode.Uri.file(newProjectArgs.espMdfPath));
@@ -124,8 +125,7 @@ export class NewProjectPanel {
             message.template
           ) {
             this.createProject(
-              newProjectArgs.espIdfPath,
-              newProjectArgs.espIdfToolsPath,
+              newProjectArgs.espIdfSetup,
               JSON.parse(message.components),
               message.port,
               message.containerFolder,
@@ -199,8 +199,7 @@ export class NewProjectPanel {
   }
 
   private async createProject(
-    idfPath: string,
-    idfToolsPath: string,
+    idfSetup: IdfSetup,
     components: IComponent[],
     port: string,
     projectDirectory: string,
@@ -287,8 +286,7 @@ export class NewProjectPanel {
           );
           const settingsJson = await setCurrentSettingsInTemplate(
             settingsJsonPath,
-            idfPath,
-            idfToolsPath,
+            idfSetup,
             port,
             openOcdConfigs,
             workspaceFolder

src/newProject/utils.ts

@@ -19,11 +19,12 @@
 import { readParameter } from "../idfConfiguration";
 import { readJSON } from "fs-extra";
 import { Uri } from "vscode";
+import { IdfSetup } from "../views/setup/types";
+import { getSystemPythonFromSettings } from "../pythonManager";
 
 export async function setCurrentSettingsInTemplate(
   settingsJsonPath: string,
-  idfPathDir: string,
-  toolsPath: string,
+  idfSetup: IdfSetup,
   port: string,
   openOcdConfigs?: string,
   workspace?: Uri
@@ -32,8 +33,17 @@ export async function setCurrentSettingsInTemplate(
   const adfPathDir = readParameter("idf.espAdfPath", workspace);
   const mdfPathDir = readParameter("idf.espMdfPath", workspace);
   const isWin = process.platform === "win32" ? "Win" : "";
-  if (idfPathDir) {
-    settingsJson["idf.espIdfPath" + isWin] = idfPathDir;
+  if (idfSetup.idfPath) {
+    settingsJson["idf.espIdfPath" + isWin] = idfSetup.idfPath;
+  }
+  if (idfSetup.python) {
+    settingsJson["idf.pythonInstallPath"] = await getSystemPythonFromSettings(
+      idfSetup.python,
+      idfSetup.idfPath,
+      idfSetup.toolsPath
+    );
+  } else {
+    settingsJson["idf.pythonInstallPath"] = idfSetup.sysPythonPath;
   }
   if (adfPathDir) {
     settingsJson["idf.espAdfPath" + isWin] = adfPathDir;
@@ -50,8 +60,8 @@ export async function setCurrentSettingsInTemplate(
   if (port.indexOf("no port") === -1) {
     settingsJson["idf.port" + isWin] = port;
   }
-  if (toolsPath) {
-    settingsJson["idf.toolsPath" + isWin] = toolsPath;
+  if (idfSetup.toolsPath) {
+    settingsJson["idf.toolsPath" + isWin] = idfSetup.toolsPath;
   }
   return settingsJson;
 }

src/pythonManager.ts

@@ -141,12 +141,6 @@ export async function installPythonEnvFromIdfTools(
     });
   }
 
-  await writeParameter(
-    "idf.pythonInstallPath",
-    pythonBinPath,
-    ConfigurationTarget.Global
-  );
-
   await execProcessWithLog(
     pythonBinPath,
     [idfToolsPyPath, "install-python-env"],
@@ -309,7 +303,10 @@ export async function execProcessWithLog(
 }
 
 export async function getVirtualEnvPythonPath(workspaceFolder: Uri) {
-  let pythonPath = readParameter("idf.pythonInstallPath") as string;
+  let pythonPath = readParameter(
+    "idf.pythonInstallPath",
+    workspaceFolder
+  ) as string;
   let espIdfDir = readParameter("idf.espIdfPath", workspaceFolder) as string;
   let idfToolsDir = readParameter("idf.toolsPath", workspaceFolder) as string;
   const idfPathExists = await pathExists(espIdfDir);
@@ -327,36 +324,38 @@ export async function getVirtualEnvPythonPath(workspaceFolder: Uri) {
 }
 
 export async function getPythonPath(workspaceFolder: Uri) {
-  let sysPythonBinPath = readParameter("idf.pythonInstallPath") as string;
+  let sysPythonBinPath = readParameter(
+    "idf.pythonInstallPath",
+    workspaceFolder
+  ) as string;
   const doesSysPythonBinPathExist = await pathExists(sysPythonBinPath);
   if (!doesSysPythonBinPathExist) {
     sysPythonBinPath = await getSystemPython(workspaceFolder);
     if (sysPythonBinPath) {
       await writeParameter(
         "idf.pythonInstallPath",
         sysPythonBinPath,
-        ConfigurationTarget.Global
+        workspaceFolder
+          ? ConfigurationTarget.WorkspaceFolder
+          : ConfigurationTarget.Global,
+        workspaceFolder
       );
     }
   }
   return sysPythonBinPath;
 }
 
-export async function getSystemPython(workspaceFolder: Uri) {
-  let pythonBinPath = readParameter(
-    "idf.pythonBinPath",
-    workspaceFolder
-  ) as string;
+export async function getSystemPythonFromSettings(
+  pythonBinPath: string,
+  espIdfPath: string,
+  espIdfToolsPath: string
+) {
   const pythonBinPathExists = await pathExists(pythonBinPath);
   if (pythonBinPathExists) {
     const pythonCode = `import sys; print('{}'.format(sys.base_prefix))`;
     const args = ["-c", pythonCode];
-    const workingDir =
-      workspaceFolder && workspaceFolder.fsPath
-        ? workspaceFolder.fsPath
-        : __dirname;
     const pythonVersion = (
-      await utils.execChildProcess(pythonBinPath, args, workingDir)
+      await utils.execChildProcess(pythonBinPath, args, __dirname)
     ).replace(/(\n|\r|\r\n)/gm, "");
     const pyDir =
       process.platform === "win32" ? ["python.exe"] : ["bin", "python3"];
@@ -368,27 +367,37 @@ export async function getSystemPython(workspaceFolder: Uri) {
     const sysPythonBinPathList = await getUnixPythonList(__dirname);
     return sysPythonBinPathList.length ? sysPythonBinPathList[0] : "python3";
   } else {
-    const idfPathDir = readParameter("idf.espIdfPath", workspaceFolder);
-    const idfToolsDir = readParameter(
-      "idf.toolsPath",
-      workspaceFolder
-    ) as string;
-    const idfVersion = await utils.getEspIdfFromCMake(idfPathDir);
+    const idfVersion = await utils.getEspIdfFromCMake(espIdfPath);
     const pythonVersionToUse =
       idfVersion >= "5.0"
         ? ESP.URL.IDF_EMBED_PYTHON.VERSION
         : ESP.URL.OLD_IDF_EMBED_PYTHON.VERSION;
     const idfPyDestPath = join(
-      idfToolsDir,
+      espIdfToolsPath,
       "tools",
       "idf-python",
       pythonVersionToUse,
       "python.exe"
     );
-    return idfPyDestPath;
+    const idfPyDestExists = await pathExists(idfPyDestPath);
+    return idfPyDestExists ? idfPyDestPath : "";
   }
 }
 
+export async function getSystemPython(workspaceFolder: Uri) {
+  let pythonBinPath = readParameter(
+    "idf.pythonBinPath",
+    workspaceFolder
+  ) as string;
+  const idfPathDir = readParameter("idf.espIdfPath", workspaceFolder);
+  const idfToolsDir = readParameter("idf.toolsPath", workspaceFolder) as string;
+  return await getSystemPythonFromSettings(
+    pythonBinPath,
+    idfPathDir,
+    idfToolsDir
+  );
+}
+
 export async function getPythonEnvPath(
   espIdfDir: string,
   idfToolsDir: string,
@@ -412,8 +421,8 @@ export async function getPythonEnvPath(
       ? ["Scripts", "python.exe"]
       : ["bin", "python"];
   const fullIdfPyEnvPath = join(idfPyEnvPath, ...pyDir);
-
-  return fullIdfPyEnvPath;
+  const pyEnvPathExists = await pathExists(fullIdfPyEnvPath);
+  return pyEnvPathExists ? fullIdfPyEnvPath : "";
 }
 
 export async function checkPythonExists(pythonBin: string, workingDir: string) {

src/setup/existingIdfSetups.ts

@@ -59,6 +59,7 @@ export async function clearPreviousIdfSetups() {
 export async function createIdfSetup(
   idfPath: string,
   toolsPath: string,
+  sysPythonBinPath: string,
   gitPath: string
 ) {
   const idfSetupId = getIdfMd5sum(idfPath);
@@ -68,6 +69,7 @@ export async function createIdfSetup(
     idfPath,
     gitPath,
     toolsPath,
+    sysPythonPath: sysPythonBinPath,
     version: idfVersion,
     isValid: false,
   };

src/setup/pyReqsInstallStep.ts

@@ -19,7 +19,6 @@ import { SetupPanel } from "./SetupPanel";
 import { saveSettings } from "./setupInit";
 import { getOpenOcdRules } from "./addOpenOcdRules";
 import { addIdfPath } from "./espIdfJson";
-import { writeParameter } from "../idfConfiguration";
 
 export async function createPyReqs(
   idfPath: string,
@@ -46,15 +45,11 @@ export async function createPyReqs(
     progress,
     cancelToken
   );
-  await writeParameter(
-    "idf.pythonInstallPath",
-    pyPath,
-    vscode.ConfigurationTarget.Global
-  );
   await saveSettings(
     idfPath,
     toolsPath,
     gitPath,
+    pyPath,
     saveScope,
     workspaceFolderUri,
     espIdfStatusBar