diff --git a/OpenRA.Mods.Common/Widgets/ViewportControllerWidget.cs b/OpenRA.Mods.Common/Widgets/ViewportControllerWidget.cs
index 598c52105a..f5b22b6a5a 100644
--- a/OpenRA.Mods.Common/Widgets/ViewportControllerWidget.cs
+++ b/OpenRA.Mods.Common/Widgets/ViewportControllerWidget.cs
@@ -349,13 +349,24 @@ namespace OpenRA.Mods.Common.Widgets
 						return true;
 				}
 
-				if (mi.Event == MouseInputEvent.Move && joystickScrollStart.HasValue)
+				if (mi.Event == MouseInputEvent.Move)
+				{
+					if (!joystickScrollStart.HasValue)
+						joystickScrollStart = mi.Location;
+
 					joystickScrollEnd = mi.Location;
+				}
 			}
 
 			return false;
 		}
 
+		public override bool YieldMouseFocus(MouseInput mi)
+		{
+			joystickScrollStart = joystickScrollEnd = null;
+			return base.YieldMouseFocus(mi);
+		}
+
 		public override bool YieldKeyboardFocus()
 		{
 			keyboardDirections = ScrollDirection.None;