diff --git a/src/gui/draw.rs b/src/gui/draw.rs
index e3dac7e..0f1748e 100644
--- a/src/gui/draw.rs
+++ b/src/gui/draw.rs
@@ -337,7 +337,7 @@ impl SoundpadGui {
                         let delete_dir_button_response =
                             ui.add_sized([18.0, 18.0], delete_dir_button);
                         if delete_dir_button_response.clicked() {
-                            self.remove_dir(&path.clone());
+                            self.app_state.dirs_to_remove.insert(path.clone());
                         }
                     });
                 });
diff --git a/src/gui/mod.rs b/src/gui/mod.rs
index bc93999..5f37d16 100644
--- a/src/gui/mod.rs
+++ b/src/gui/mod.rs
@@ -96,18 +96,6 @@ impl SoundpadGui {
         }
     }
 
-    pub fn remove_dir(&mut self, path: &PathBuf) {
-        self.app_state.dirs.retain(|x| x != path);
-        if let Some(current_dir) = &self.app_state.current_dir
-            && current_dir == path
-        {
-            self.app_state.current_dir = None;
-            self.app_state.files.clear();
-        }
-        self.config.dirs = self.app_state.dirs.clone();
-        self.config.save_to_file().ok();
-    }
-
     pub fn open_dir(&mut self, path: &PathBuf) {
         self.app_state.current_dir = Some(path.clone());
         match path.read_dir() {
diff --git a/src/gui/update.rs b/src/gui/update.rs
index af55de6..052de94 100644
--- a/src/gui/update.rs
+++ b/src/gui/update.rs
@@ -9,6 +9,17 @@ use std::time::{Duration, Instant};
 
 impl App for SoundpadGui {
     fn update(&mut self, ctx: &Context, _frame: &mut EFrame) {
+        // Remove directories
+        for path in self.app_state.dirs_to_remove.drain() {
+            self.app_state.dirs.retain(|x| x != &path);
+            if let Some(current_dir) = &self.app_state.current_dir
+                && current_dir == &path
+            {
+                self.app_state.current_dir = None;
+                self.app_state.files.clear();
+            }
+        }
+
         // Save directories if changed
         if !self.config.dirs.eq(&self.app_state.dirs) {
             self.config.dirs = self.app_state.dirs.clone();
diff --git a/src/types/gui.rs b/src/types/gui.rs
index 74db570..8e4cb1d 100644
--- a/src/types/gui.rs
+++ b/src/types/gui.rs
@@ -38,6 +38,7 @@ pub struct AppState {
 
     pub current_dir: Option<PathBuf>,
     pub dirs: Vec<PathBuf>,
+    pub dirs_to_remove: HashSet<PathBuf>,
 
     pub selected_file: Option<PathBuf>,
     pub files: HashSet<PathBuf>,