diff --git a/OpenRA.Editor/TileSetRenderer.cs b/OpenRA.Editor/TileSetRenderer.cs
index c1bcef7c46..ab23986121 100644
--- a/OpenRA.Editor/TileSetRenderer.cs
+++ b/OpenRA.Editor/TileSetRenderer.cs
@@ -22,7 +22,7 @@ namespace OpenRA.Editor
 	{
 		public readonly int TileSize;
 		public TileSet TileSet;
-		Dictionary<ushort, List<byte[]>> templates;
+		Dictionary<ushort, byte[][]> templates;
 
 		// Extract a square tile that the editor can render
 		byte[] ExtractSquareTile(ISpriteFrame frame)
@@ -44,42 +44,19 @@ namespace OpenRA.Editor
 			return data;
 		}
 
-		List<byte[]> LoadTemplate(string filename, string[] exts, Dictionary<string, ISpriteSource> sourceCache, int[] frames)
-		{
-			ISpriteSource source;
-			if (!sourceCache.ContainsKey(filename))
-			{
-				using (var s = GlobalFileSystem.OpenWithExts(filename, exts))
-					source = SpriteSource.LoadSpriteSource(s, filename);
-
-				if (source.CacheWhenLoadingTileset)
-					sourceCache.Add(filename, source);
-			}
-			else
-				source = sourceCache[filename];
-
-			if (frames != null)
-			{
-				var ret = new List<byte[]>();
-				var srcFrames = source.Frames;
-				foreach (var i in frames)
-					ret.Add(ExtractSquareTile(srcFrames[i]));
-
-				return ret;
-			}
-
-			return source.Frames.Select(f => ExtractSquareTile(f)).ToList();
-		}
-
 		public TileSetRenderer(TileSet tileset, Size tileSize)
 		{
 			this.TileSet = tileset;
 			this.TileSize = Math.Min(tileSize.Width, tileSize.Height);
 
-			templates = new Dictionary<ushort, List<byte[]>>();
-			var sourceCache = new Dictionary<string, ISpriteSource>();
-			foreach (var t in TileSet.Templates)
-				templates.Add(t.Key, LoadTemplate(t.Value.Image, tileset.Extensions, sourceCache, t.Value.Frames));
+			templates = new Dictionary<ushort, byte[][]>();
+			var spriteLoader = new SpriteLoader(tileset.Extensions, null);
+			foreach (var t in tileset.Templates)
+			{
+				var allFrames = spriteLoader.LoadAllFrames(t.Value.Image);
+				var frames = t.Value.Frames != null ? t.Value.Frames.Select(f => allFrames[f]).ToArray() : allFrames;
+				templates.Add(t.Value.Id, frames.Select(f => ExtractSquareTile(f)).ToArray());
+			}
 		}
 
 		public Bitmap RenderTemplate(ushort id, IPalette p)
diff --git a/OpenRA.Game/FileFormats/R8Reader.cs b/OpenRA.Game/FileFormats/R8Reader.cs
index 0db3ff57ff..1448ee25b6 100644
--- a/OpenRA.Game/FileFormats/R8Reader.cs
+++ b/OpenRA.Game/FileFormats/R8Reader.cs
@@ -62,7 +62,6 @@ namespace OpenRA.FileFormats
 		}
 
 		public IReadOnlyList<ISpriteFrame> Frames { get; private set; }
-		public bool CacheWhenLoadingTileset { get { return true; } }
 
 		public readonly int ImageCount;
 		public R8Reader(Stream stream)
diff --git a/OpenRA.Game/FileFormats/ShpD2Reader.cs b/OpenRA.Game/FileFormats/ShpD2Reader.cs
index dca7a888d6..51231eb618 100644
--- a/OpenRA.Game/FileFormats/ShpD2Reader.cs
+++ b/OpenRA.Game/FileFormats/ShpD2Reader.cs
@@ -84,7 +84,6 @@ namespace OpenRA.FileFormats
 		}
 
 		public IReadOnlyList<ISpriteFrame> Frames { get; private set; }
-		public bool CacheWhenLoadingTileset { get { return false; } }
 
 		public ShpD2Reader(Stream s)
 		{
diff --git a/OpenRA.Game/FileFormats/ShpReader.cs b/OpenRA.Game/FileFormats/ShpReader.cs
index 0d7004cca5..173deb5ed6 100644
--- a/OpenRA.Game/FileFormats/ShpReader.cs
+++ b/OpenRA.Game/FileFormats/ShpReader.cs
@@ -60,7 +60,6 @@ namespace OpenRA.FileFormats
 		}
 
 		public IReadOnlyList<ISpriteFrame> Frames { get; private set; }
-		public bool CacheWhenLoadingTileset { get { return false; } }
 		public readonly Size Size;
 
 		int recurseDepth = 0;
diff --git a/OpenRA.Game/FileFormats/ShpTSReader.cs b/OpenRA.Game/FileFormats/ShpTSReader.cs
index 661a2790f4..6b067f0fcb 100644
--- a/OpenRA.Game/FileFormats/ShpTSReader.cs
+++ b/OpenRA.Game/FileFormats/ShpTSReader.cs
@@ -45,7 +45,6 @@ namespace OpenRA.FileFormats
 		}
 
 		public IReadOnlyList<ISpriteFrame> Frames { get; private set; }
-		public bool CacheWhenLoadingTileset { get { return false; } }
 
 		public ShpTSReader(Stream stream)
 		{
diff --git a/OpenRA.Game/FileFormats/TmpRAReader.cs b/OpenRA.Game/FileFormats/TmpRAReader.cs
index 5b8f4b520e..cfd761e84a 100644
--- a/OpenRA.Game/FileFormats/TmpRAReader.cs
+++ b/OpenRA.Game/FileFormats/TmpRAReader.cs
@@ -17,7 +17,6 @@ namespace OpenRA.FileFormats
 	public class TmpRAReader : ISpriteSource
 	{
 		public IReadOnlyList<ISpriteFrame> Frames { get; private set; }
-		public bool CacheWhenLoadingTileset { get { return false; } }
 
 		public TmpRAReader(Stream s)
 		{
diff --git a/OpenRA.Game/FileFormats/TmpTDReader.cs b/OpenRA.Game/FileFormats/TmpTDReader.cs
index d0521460d2..540d63c6af 100644
--- a/OpenRA.Game/FileFormats/TmpTDReader.cs
+++ b/OpenRA.Game/FileFormats/TmpTDReader.cs
@@ -36,7 +36,6 @@ namespace OpenRA.FileFormats
 	public class TmpTDReader : ISpriteSource
 	{
 		public IReadOnlyList<ISpriteFrame> Frames { get; private set; }
-		public bool CacheWhenLoadingTileset { get { return false; } }
 
 		public TmpTDReader(Stream s)
 		{
diff --git a/OpenRA.Game/FileFormats/TmpTSReader.cs b/OpenRA.Game/FileFormats/TmpTSReader.cs
index d405e2d01d..e6fdb4a7b9 100644
--- a/OpenRA.Game/FileFormats/TmpTSReader.cs
+++ b/OpenRA.Game/FileFormats/TmpTSReader.cs
@@ -50,7 +50,6 @@ namespace OpenRA.FileFormats
 	public class TmpTSReader : ISpriteSource
 	{
 		public IReadOnlyList<ISpriteFrame> Frames { get; private set; }
-		public bool CacheWhenLoadingTileset { get { return false; } }
 
 		public TmpTSReader(Stream s)
 		{
diff --git a/OpenRA.Game/Graphics/SpriteLoader.cs b/OpenRA.Game/Graphics/SpriteLoader.cs
index fe7c8781b5..5414e6c7b5 100644
--- a/OpenRA.Game/Graphics/SpriteLoader.cs
+++ b/OpenRA.Game/Graphics/SpriteLoader.cs
@@ -18,6 +18,7 @@ namespace OpenRA.Graphics
 	{
 		public readonly SheetBuilder SheetBuilder;
 		readonly Cache<string, Sprite[]> sprites;
+		readonly Cache<string, ISpriteFrame[]> frames;
 		readonly string[] exts;
 
 		public SpriteLoader(string[] exts, SheetBuilder sheetBuilder)
@@ -26,17 +27,24 @@ namespace OpenRA.Graphics
 
 			// Include extension-less version
 			this.exts = exts.Append("").ToArray();
-			sprites = new Cache<string, Sprite[]>(CacheSpriteFrames);
+			sprites = new Cache<string, Sprite[]>(CacheSprites);
+			frames = new Cache<string, ISpriteFrame[]>(CacheFrames);
 		}
 
-		Sprite[] CacheSpriteFrames(string filename)
+		Sprite[] CacheSprites(string filename)
+		{
+			return frames[filename].Select(a => SheetBuilder.Add(a))
+				.ToArray();
+		}
+
+		ISpriteFrame[] CacheFrames(string filename)
 		{
 			using (var stream = GlobalFileSystem.OpenWithExts(filename, exts))
 				return SpriteSource.LoadSpriteSource(stream, filename).Frames
-					.Select(a => SheetBuilder.Add(a))
 					.ToArray();
 		}
 
 		public Sprite[] LoadAllSprites(string filename) { return sprites[filename]; }
+		public ISpriteFrame[] LoadAllFrames(string filename) { return frames[filename]; }
 	}
 }
diff --git a/OpenRA.Game/Graphics/SpriteSource.cs b/OpenRA.Game/Graphics/SpriteSource.cs
index ae23aa5501..0434fd1bde 100644
--- a/OpenRA.Game/Graphics/SpriteSource.cs
+++ b/OpenRA.Game/Graphics/SpriteSource.cs
@@ -25,7 +25,6 @@ namespace OpenRA.Graphics
 	public interface ISpriteSource
 	{
 		IReadOnlyList<ISpriteFrame> Frames { get; }
-		bool CacheWhenLoadingTileset { get; }
 	}
 
 	// TODO: Most of this should be moved into the format parsers themselves.
diff --git a/OpenRA.Game/Graphics/Theater.cs b/OpenRA.Game/Graphics/Theater.cs
index e3b962160e..a5b87b04be 100644
--- a/OpenRA.Game/Graphics/Theater.cs
+++ b/OpenRA.Game/Graphics/Theater.cs
@@ -22,33 +22,6 @@ namespace OpenRA.Graphics
 		Dictionary<ushort, Sprite[]> templates;
 		Sprite missingTile;
 
-		Sprite[] LoadTemplate(string filename, string[] exts, Dictionary<string, ISpriteSource> sourceCache, int[] frames)
-		{
-			ISpriteSource source;
-			if (!sourceCache.ContainsKey(filename))
-			{
-				using (var s = GlobalFileSystem.OpenWithExts(filename, exts))
-					source = SpriteSource.LoadSpriteSource(s, filename);
-
-				if (source.CacheWhenLoadingTileset)
-					sourceCache.Add(filename, source);
-			}
-			else
-				source = sourceCache[filename];
-
-			if (frames != null)
-			{
-				var ret = new List<Sprite>();
-				var srcFrames = source.Frames.ToArray();
-				foreach (var i in frames)
-					ret.Add(sheetBuilder.Add(srcFrames[i]));
-
-				return ret.ToArray();
-			}
-
-			return source.Frames.Select(f => sheetBuilder.Add(f)).ToArray();
-		}
-
 		public Theater(TileSet tileset)
 		{
 			var allocated = false;
@@ -61,11 +34,17 @@ namespace OpenRA.Graphics
 				return new Sheet(new Size(tileset.SheetSize, tileset.SheetSize), true);
 			};
 
-			var sourceCache = new Dictionary<string, ISpriteSource>();
-			templates = new Dictionary<ushort, Sprite[]>();
 			sheetBuilder = new SheetBuilder(SheetType.Indexed, allocate);
+			templates = new Dictionary<ushort, Sprite[]>();
+
+			// We manage the SheetBuilder ourselves, to avoid loading all of the tileset images
+			var spriteLoader = new SpriteLoader(tileset.Extensions, null);
 			foreach (var t in tileset.Templates)
-				templates.Add(t.Value.Id, LoadTemplate(t.Value.Image, tileset.Extensions, sourceCache, t.Value.Frames));
+			{
+				var allFrames = spriteLoader.LoadAllFrames(t.Value.Image);
+				var frames = t.Value.Frames != null ? t.Value.Frames.Select(f => allFrames[f]).ToArray() : allFrames;
+				templates.Add(t.Value.Id, frames.Select(f => sheetBuilder.Add(f)).ToArray());
+			}
 
 			// 1x1px transparent tile
 			missingTile = sheetBuilder.Add(new byte[1], new Size(1, 1));