diff --git a/OpenRA.Game/Actor.cs b/OpenRA.Game/Actor.cs
index 58ff4eaf1a..4d8247ae43 100644
--- a/OpenRA.Game/Actor.cs
+++ b/OpenRA.Game/Actor.cs
@@ -242,12 +242,17 @@ namespace OpenRA
 					return;
 
 				var oldOwner = Owner;
+				var wasInWorld = IsInWorld;
 
 				// momentarily remove from world so the ownership queries don't get confused
-				w.Remove(this);
+				if (wasInWorld)
+					w.Remove(this);
+
 				Owner = newOwner;
 				Generation++;
-				w.Add(this);
+
+				if (wasInWorld)
+					w.Add(this);
 
 				foreach (var t in this.TraitsImplementing<INotifyOwnerChanged>())
 					t.OnOwnerChanged(this, oldOwner, newOwner);