diff --git a/src/layout/scrolling.rs b/src/layout/scrolling.rs
index 84ad835eb..635f4aeff 100644
--- a/src/layout/scrolling.rs
+++ b/src/layout/scrolling.rs
@@ -1795,7 +1795,6 @@ impl<W: LayoutElement> ScrollingSpace<W> {
 
         let source_tile_idx = self.columns[source_column_idx].active_tile_idx;
         let target_tile_idx = self.columns[target_column_idx].active_tile_idx;
-        let source_column_drained = self.columns[source_column_idx].tiles.len() == 1;
 
         // capture the original positions of the tiles
         let (mut source_pt, mut target_pt) = (
@@ -1807,80 +1806,30 @@ impl<W: LayoutElement> ScrollingSpace<W> {
         source_pt.x += self.column_x(source_column_idx);
         target_pt.x += self.column_x(target_column_idx);
 
-        let transaction = Transaction::new();
-
-        // If the source column contains a single tile, this will also remove the column.
-        // When this happens `source_column_drained` will be set and the column will need to be
-        // recreated with `add_tile`
-        let source_removed = self.remove_tile_by_idx(
-            source_column_idx,
-            source_tile_idx,
-            transaction.clone(),
-            None,
+        // swap the two tiles directly in their respective column structs
+        let (source_col, target_col) = if source_column_idx < target_column_idx {
+            let (a, b) = self.columns.split_at_mut(target_column_idx);
+            (&mut a[source_column_idx], &mut b[0])
+        } else {
+            let (a, b) = self.columns.split_at_mut(source_column_idx);
+            (&mut b[0], &mut a[target_column_idx])
+        };
+        std::mem::swap(
+            &mut source_col.tiles[source_tile_idx],
+            &mut target_col.tiles[target_tile_idx],
         );
 
-        {
-            // special case when the source column disappears after removing its last tile
-            let adjusted_target_column_idx =
-                if direction == ScrollDirection::Right && source_column_drained {
-                    target_column_idx - 1
-                } else {
-                    target_column_idx
-                };
-
-            self.add_tile_to_column(
-                adjusted_target_column_idx,
-                Some(target_tile_idx),
-                source_removed.tile,
-                false,
-            );
-
-            let RemovedTile {
-                tile: target_tile, ..
-            } = self.remove_tile_by_idx(
-                adjusted_target_column_idx,
-                target_tile_idx + 1,
-                transaction.clone(),
-                None,
-            );
-
-            if source_column_drained {
-                // recreate the drained column with only the target tile
-                self.add_tile(
-                    Some(source_column_idx),
-                    target_tile,
-                    true,
-                    source_removed.width,
-                    source_removed.is_full_width,
-                    None,
-                )
-            } else {
-                // simply add the removed target tile to the source column
-                self.add_tile_to_column(
-                    source_column_idx,
-                    Some(source_tile_idx),
-                    target_tile,
-                    false,
-                );
-            }
-        }
-
-        // update the active tile in the modified columns
-        self.columns[source_column_idx].active_tile_idx = source_tile_idx;
-        self.columns[target_column_idx].active_tile_idx = target_tile_idx;
-
         // Animations
-        self.columns[target_column_idx].tiles[target_tile_idx]
-            .animate_move_from(source_pt - target_pt);
-
-        // FIXME: this stop_move_animations() causes the target tile animation to "reset" when
-        // swapping. It's here as a workaround to stop the unwanted animation of moving the source
-        // tile down when adding the target tile above it. This code needs to be written in some
-        // other way not to trigger that animation, or to cancel it properly, so that swap doesn't
-        // cancel all ongoing target tile animations.
-        self.columns[source_column_idx].tiles[source_tile_idx].stop_move_animations();
-        self.columns[source_column_idx].tiles[source_tile_idx]
-            .animate_move_from(target_pt - source_pt);
+        let (source_tile, target_tile) = (
+            &mut target_col.tiles[target_tile_idx],
+            &mut source_col.tiles[source_tile_idx],
+        );
+        source_tile.animate_move_from(source_pt - target_pt);
+        target_tile.animate_move_from(target_pt - source_pt);
+
+        let transaction = Transaction::new();
+        source_tile.request_tile_size(target_tile.tile_size(), true, Some(transaction.clone()));
+        target_tile.request_tile_size(source_tile.tile_size(), true, Some(transaction));
 
         self.activate_column(target_column_idx);
     }