fix(gui): reduce threads count on low memory, other tweaks (#1410)

jadx-gui/src/main/java/jadx/gui/JadxWrapper.java

@@ -16,9 +16,14 @@
 import jadx.api.JavaClass;
 import jadx.api.JavaPackage;
 import jadx.api.ResourceFile;
+import jadx.core.dex.nodes.ClassNode;
+import jadx.core.dex.nodes.ProcessState;
 import jadx.gui.settings.JadxProject;
 import jadx.gui.settings.JadxSettings;
 
+import static jadx.core.dex.nodes.ProcessState.GENERATED_AND_UNLOADED;
+import static jadx.core.dex.nodes.ProcessState.NOT_LOADED;
+import static jadx.core.dex.nodes.ProcessState.PROCESS_COMPLETE;
 import static jadx.gui.utils.FileUtils.toFiles;
 
 public class JadxWrapper {
@@ -51,6 +56,15 @@ public void openFile(List<Path> paths) {
 		}
 	}
 
+	// TODO: check and move into core package
+	public void unloadClasses() {
+		for (ClassNode cls : decompiler.getRoot().getClasses()) {
+			ProcessState clsState = cls.getState();
+			cls.unload();
+			cls.setState(clsState == PROCESS_COMPLETE ? GENERATED_AND_UNLOADED : NOT_LOADED);
+		}
+	}
+
 	public void close() {
 		try {
 			decompiler.close();

jadx-gui/src/main/java/jadx/gui/jobs/BackgroundExecutor.java

@@ -76,8 +76,8 @@ public void execute(String title, Runnable backgroundRunnable, Consumer<TaskStat
 		execute(new SimpleTask(title, Collections.singletonList(backgroundRunnable), onFinishUiRunnable));
 	}
 
-	public void execute(String title, Runnable backgroundRunnable) {
-		execute(new SimpleTask(title, Collections.singletonList(backgroundRunnable), null));
+	public Future<TaskStatus> execute(String title, Runnable backgroundRunnable) {
+		return execute(new SimpleTask(title, Collections.singletonList(backgroundRunnable), null));
 	}
 
 	private ThreadPoolExecutor makeTaskQueueExecutor() {
@@ -86,6 +86,7 @@ private ThreadPoolExecutor makeTaskQueueExecutor() {
 
 	private final class TaskWorker extends SwingWorker<TaskStatus, Void> implements ITaskInfo {
 		private final IBackgroundTask task;
+		private ThreadPoolExecutor executor;
 		private TaskStatus status = TaskStatus.WAIT;
 		private long jobsCount;
 		private long jobsComplete;
@@ -117,7 +118,7 @@ private void runJobs() throws InterruptedException {
 					task.getTitle(), jobsCount, task.timeLimit(), task.checkMemoryUsage());
 			status = TaskStatus.STARTED;
 			int threadsCount = mainWindow.getSettings().getThreadsCount();
-			ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(threadsCount);
+			executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(threadsCount);
 			for (Runnable job : jobs) {
 				executor.execute(job);
 			}
@@ -174,14 +175,22 @@ private Supplier<TaskStatus> buildCancelCheck(long startTime) {
 					LOG.error("Task '{}' execution timeout, force cancel", task.getTitle());
 					return TaskStatus.CANCEL_BY_TIMEOUT;
 				}
-				if (checkMemoryUsage && !UiUtils.isFreeMemoryAvailable()) {
-					LOG.error("Task '{}' memory limit reached, force cancel", task.getTitle());
-					return TaskStatus.CANCEL_BY_MEMORY;
-				}
 				if (isCancelled() || Thread.currentThread().isInterrupted()) {
 					LOG.warn("Task '{}' canceled", task.getTitle());
 					return TaskStatus.CANCEL_BY_USER;
 				}
+				if (checkMemoryUsage && !UiUtils.isFreeMemoryAvailable()) {
+					LOG.info("Memory usage: {}", UiUtils.memoryInfo());
+					if (executor.getCorePoolSize() == 1) {
+						LOG.error("Task '{}' memory limit reached, force cancel", task.getTitle());
+						return TaskStatus.CANCEL_BY_MEMORY;
+					}
+					LOG.warn("Low memory, reduce processing threads count to 1");
+					// reduce thread count and continue
+					executor.setCorePoolSize(1);
+					System.gc();
+					UiUtils.sleep(500); // wait GC
+				}
 				return null;
 			};
 		}

jadx-gui/src/main/java/jadx/gui/ui/MainWindow.java

@@ -85,6 +85,7 @@
 import jadx.api.JadxArgs;
 import jadx.api.JavaNode;
 import jadx.api.ResourceFile;
+import jadx.api.plugins.utils.CommonFileUtils;
 import jadx.core.Jadx;
 import jadx.core.utils.StringUtils;
 import jadx.core.utils.Utils;
@@ -342,9 +343,7 @@ private void newProject() {
 		if (!ensureProjectIsSaved()) {
 			return;
 		}
-		cancelBackgroundJobs();
-		clearTree();
-		wrapper.close();
+		closeAll();
 		updateProject(new JadxProject());
 	}
 
@@ -406,16 +405,14 @@ void open(List<Path> paths) {
 	void open(List<Path> paths, Runnable onFinish) {
 		if (paths.size() == 1) {
 			Path singleFile = paths.get(0);
-			if (singleFile.getFileName().toString().toLowerCase(Locale.ROOT).endsWith(JadxProject.PROJECT_EXTENSION)) {
-				openProject(singleFile);
-				onFinish.run();
+			String fileExtension = CommonFileUtils.getFileExtension(singleFile.getFileName().toString());
+			if (fileExtension != null && fileExtension.equalsIgnoreCase(JadxProject.PROJECT_EXTENSION)) {
+				openProject(singleFile, onFinish);
 				return;
 			}
 		}
+		closeAll();
 		project.setFilePath(paths);
-		clearTree();
-		BreakpointManager.saveAndExit();
-		LogCollector.getInstance().reset();
 		if (paths.isEmpty()) {
 			return;
 		}
@@ -433,6 +430,17 @@ void open(List<Path> paths, Runnable onFinish) {
 				});
 	}
 
+	private void closeAll() {
+		cancelBackgroundJobs();
+		saveOpenTabs();
+		clearTree();
+		BreakpointManager.saveAndExit();
+		LogCollector.getInstance().reset();
+		wrapper.close();
+		tabbedPane.closeAllTabs();
+		System.gc();
+	}
+
 	private void checkLoadedStatus() {
 		if (!wrapper.getClasses().isEmpty()) {
 			return;
@@ -483,7 +491,7 @@ private boolean ensureProjectIsSaved() {
 		return true;
 	}
 
-	private void openProject(Path path) {
+	private void openProject(Path path, Runnable onFinish) {
 		if (!ensureProjectIsSaved()) {
 			return;
 		}
@@ -500,9 +508,9 @@ private void openProject(Path path) {
 		settings.addRecentProject(path);
 		List<Path> filePaths = jadxProject.getFilePaths();
 		if (filePaths == null) {
-			clearTree();
+			closeAll();
 		} else {
-			open(filePaths);
+			open(filePaths, onFinish);
 		}
 	}
 
@@ -561,10 +569,14 @@ public void waitDecompileTask() {
 				DecompileTask decompileTask = new DecompileTask(this, wrapper);
 				backgroundExecutor.executeAndWait(decompileTask);
 
+				backgroundExecutor.execute(decompileTask.getTitle(), wrapper::unloadClasses).get();
+				System.gc();
+
 				IndexTask indexTask = new IndexTask(this, wrapper);
 				backgroundExecutor.executeAndWait(indexTask);
 
 				processDecompilationResults(decompileTask.getResult(), indexTask.getResult());
+				System.gc();
 			} catch (Exception e) {
 				LOG.error("Decompile task execution failed", e);
 			}
@@ -1395,7 +1407,9 @@ private void closeWindow() {
 	}
 
 	private void saveOpenTabs() {
-		project.saveOpenTabs(tabbedPane.getEditorViewStates(), tabbedPane.getSelectedIndex());
+		if (project != null) {
+			project.saveOpenTabs(tabbedPane.getEditorViewStates(), tabbedPane.getSelectedIndex());
+		}
 	}
 
 	private void restoreOpenTabs() {

jadx-gui/src/main/java/jadx/gui/utils/UiUtils.java

@@ -314,4 +314,12 @@ public static void copyToClipboard(String text) {
 			LOG.error("Failed copy text to clipboard", e);
 		}
 	}
+
+	public static void sleep(int ms) {
+		try {
+			Thread.sleep(ms);
+		} catch (InterruptedException e) {
+			// ignore
+		}
+	}
 }