diff --git a/src/audio/module_adapter/module/modules.c b/src/audio/module_adapter/module/modules.c
index dcf9d109fe30..c1a93f2811f8 100644
--- a/src/audio/module_adapter/module/modules.c
+++ b/src/audio/module_adapter/module/modules.c
@@ -14,6 +14,7 @@
 #include <sof/lib_manager.h>
 #include <sof/audio/module_adapter/module/module_interface.h>
 #include <module/module/api_ver.h>
+#include <zephyr/llext/llext.h>
 
 /* Intel module adapter is an extension to SOF module adapter component that allows to integrate
  * modules developed under IADK (Intel Audio Development Kit) Framework. IADK modules uses uniform
@@ -61,20 +62,23 @@ static int modules_init(struct processing_module *mod)
 	struct module_data *md = &mod->priv;
 	struct comp_dev *dev = mod->dev;
 	const struct ipc4_base_module_cfg *src_cfg = &md->cfg.base_cfg;
+	struct comp_ipc_config *config = &mod->dev->ipc_config;
+	const void *buildinfo = NULL;
+	/*
+	 * Allocate module resources and move it to L2 memory, or just increment
+	 * the use-count.
+	 */
+	uintptr_t module_entry_point = lib_manager_allocate_module(mod, config,
+								   src_cfg, &buildinfo);
+
+	if (module_entry_point == 0) {
+		comp_err(dev, "modules_init(), lib_manager_allocate_module() failed!");
+		return -EINVAL;
+	}
+	comp_info(mod->dev, "modules_init() start");
 
 	if (!md->module_adapter) {
-		struct comp_ipc_config *config = &(mod->dev->ipc_config);
-		/* At this point module resources are allocated and it is moved to L2 memory. */
-		const void *buildinfo = NULL;
-		uintptr_t module_entry_point = lib_manager_allocate_module(mod, config,
-									   src_cfg, &buildinfo);
-
-		if (module_entry_point == 0) {
-			comp_err(dev, "modules_init(), lib_manager_allocate_module() failed!");
-			return -EINVAL;
-		}
-		comp_info(mod->dev, "modules_init() start");
-
+		/* First load */
 		uint32_t module_id = IPC4_MOD_ID(config->id);
 		uint32_t instance_id = IPC4_INST_ID(config->id);
 		uint32_t log_handle = (uint32_t)mod->dev->drv->tctx;
@@ -309,6 +313,18 @@ static int modules_free(struct processing_module *mod)
 	rfree(md->mpd.in_buff);
 	rfree(md->mpd.out_buff);
 
+	if (md->llext) {
+		/* This uses a reference counter to decide, when to unload */
+		int cnt = llext_unload(&md->llext);
+
+		if (!cnt) {
+			/* Free module resources allocated in L2 memory. */
+			ret = lib_manager_free_module(mod, &dev->ipc_config);
+			if (ret < 0)
+				comp_err(dev, "modules_free(), lib_manager_free_module() failed!");
+		}
+	}
+
 	return ret;
 }
 
diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h
index 6d28cb657931..64d211866fab 100644
--- a/src/include/sof/lib_manager.h
+++ b/src/include/sof/lib_manager.h
@@ -150,7 +150,7 @@ uintptr_t lib_manager_allocate_module(struct processing_module *proc,
  * Function is responsible to free module resources in HP memory.
  */
 int lib_manager_free_module(struct processing_module *proc,
-			    struct comp_ipc_config *ipc_config);
+			    const struct comp_ipc_config *ipc_config);
 /*
  * \brief Load library
  *
diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c
index a345c5827f5a..eadd0d757b9d 100644
--- a/src/library_manager/lib_manager.c
+++ b/src/library_manager/lib_manager.c
@@ -150,9 +150,7 @@ static int lib_manager_allocate_module_instance(uint32_t module_id, uint32_t ins
 	void __sparse_cache *va_base = lib_manager_get_instance_bss_address(module_id,
 									    instance_id, mod);
 
-	/*
-	 * Map bss memory and clear it.
-	 */
+	/* Map bss memory and clear it. */
 	if (sys_mm_drv_map_region((__sparse_force void *)va_base, POINTER_TO_UINT(NULL),
 				  bss_size, SYS_MM_MEM_PERM_RW) < 0)
 		return -ENOMEM;
@@ -168,9 +166,7 @@ static int lib_manager_free_module_instance(uint32_t module_id, uint32_t instanc
 	uint32_t bss_size = mod->segment[SOF_MAN_SEGMENT_BSS].flags.r.length * PAGE_SZ;
 	void __sparse_cache *va_base = lib_manager_get_instance_bss_address(module_id,
 									    instance_id, mod);
-	/*
-	 * Unmap bss memory.
-	 */
+	/* Unmap bss memory. */
 	return sys_mm_drv_unmap_region((__sparse_force void *)va_base, bss_size);
 }
 
@@ -195,8 +191,14 @@ static int lib_manager_link_llext(struct sof_man_fw_desc *desc, struct sof_man_m
 	struct llext_load_param ldr_parm = {false};
 	int ret = llext_load(&ebl.loader, mod->name, &md->llext, &ldr_parm);
 
-	if (ret < 0)
+	if (ret > 0)
+		/* Already loaded */
+		return md->module_entry_point;
+	if (ret < 0) {
+		tr_err(&lib_manager_tr,
+		       "llext loading failed: %d", ret);
 		return ret;
+	}
 
 	mod->segment[SOF_MAN_SEGMENT_TEXT].v_base_addr = ebl.loader.sects[LLEXT_SECT_TEXT].sh_addr;
 	mod->segment[SOF_MAN_SEGMENT_TEXT].flags.r.length =
@@ -276,20 +278,27 @@ uintptr_t lib_manager_allocate_module(struct processing_module *proc,
 	/* Map .text and the rest as .data */
 	ret = lib_manager_load_module(module_id, mod, desc);
 	if (ret < 0)
-		return 0;
+		goto e_ldmod;
 
 	ret = lib_manager_allocate_module_instance(module_id, IPC4_INST_ID(ipc_config->id), mod);
 	if (ret < 0) {
 		tr_err(&lib_manager_tr,
 		       "lib_manager_allocate_module(): module allocation failed: %d", ret);
-		return 0;
+		goto e_bss;
 	}
 
 	return mod_manifest->module.entry_point;
+
+e_bss:
+	lib_manager_unload_module(module_id, mod, desc);
+e_ldmod:
+	if (module_is_llext(mod))
+		llext_unload(&proc->priv.llext);
+	return 0;
 }
 
 int lib_manager_free_module(struct processing_module *proc,
-			    struct comp_ipc_config *ipc_config)
+			    const struct comp_ipc_config *ipc_config)
 {
 	struct sof_man_fw_desc *desc;
 	struct sof_man_module *mod;