Merge pull request #10687 from pchote/filesystem-yaml

Load mod yaml via the virtual filesystem.

.travis.yml

@@ -10,6 +10,7 @@ sudo: false
 cache:
   directories:
   - thirdparty/download
+  - Support/Content
 
 addons:
   apt:
@@ -26,12 +27,14 @@ env:
 
 # Fetch dependencies
 # Run the build script
+# Fetch mod assets
 # Check source code with StyleCop
 # call OpenRA to check for YAML errors
 # Run the NUnit tests
 script:
  - travis_retry make all-dependencies
  - make all
+ - travis_retry ./thirdparty/fetch-travis-assets.sh
  - make check
  - make check-scripts
  - make test

Makefile

@@ -254,15 +254,19 @@ test: utility mods
 	@echo
 	@echo "Testing Tiberian Sun mod MiniYAML..."
 	@mono --debug OpenRA.Utility.exe ts --check-yaml
+	@mono --debug OpenRA.Utility.exe ts --check-sequence-sprites
 	@echo
 	@echo "Testing Dune 2000 mod MiniYAML..."
 	@mono --debug OpenRA.Utility.exe d2k --check-yaml
+	@mono --debug OpenRA.Utility.exe d2k --check-sequence-sprites
 	@echo
 	@echo "Testing Tiberian Dawn mod MiniYAML..."
 	@mono --debug OpenRA.Utility.exe cnc --check-yaml
+	@mono --debug OpenRA.Utility.exe cnc --check-sequence-sprites
 	@echo
 	@echo "Testing Red Alert mod MiniYAML..."
 	@mono --debug OpenRA.Utility.exe ra --check-yaml
+	@mono --debug OpenRA.Utility.exe ra --check-sequence-sprites
 
 
 ##### Launchers / Utilities #####

OpenRA.Game/Game.cs

@@ -362,7 +362,6 @@ namespace OpenRA
 				return;
 			}
 
-			ModData.MountFiles();
 			ModData.InitializeLoaders();
 			Renderer.InitializeFonts(ModData.Manifest);
 

OpenRA.Game/GameRules/RulesetCache.cs

@@ -112,7 +112,7 @@ namespace OpenRA
 				return t;
 			};
 
-			var tree = MiniYaml.Merge(files.Select(MiniYaml.FromFile).Append(nodes))
+			var tree = MiniYaml.Merge(files.Select(s => MiniYaml.FromStream(modData.ModFiles.Open(s))).Append(nodes))
 				.ToDictionaryWithConflictLog(n => n.Key, n => n.Value, "LoadYamlRules", null, null);
 			RaiseProgress();
 

OpenRA.Game/Graphics/ChromeProvider.cs

@@ -25,7 +25,7 @@ namespace OpenRA.Graphics
 		static Dictionary<string, Sheet> cachedSheets;
 		static Dictionary<string, Dictionary<string, Sprite>> cachedSprites;
 
-		public static void Initialize(IEnumerable<string> chromeFiles)
+		public static void Initialize(ModData modData)
 		{
 			Deinitialize();
 
@@ -33,7 +33,9 @@ namespace OpenRA.Graphics
 			cachedSheets = new Dictionary<string, Sheet>();
 			cachedSprites = new Dictionary<string, Dictionary<string, Sprite>>();
 
-			var chrome = MiniYaml.Merge(chromeFiles.Select(MiniYaml.FromFile));
+			var chrome = MiniYaml.Merge(modData.Manifest.Chrome
+				.Select(s => MiniYaml.FromStream(modData.ModFiles.Open(s))));
+
 			foreach (var c in chrome)
 				LoadCollection(c.Key, c.Value);
 		}

OpenRA.Game/Graphics/CursorProvider.cs

@@ -21,11 +21,12 @@ namespace OpenRA.Graphics
 
 		public CursorProvider(ModData modData)
 		{
-			var sequences = new MiniYaml(null, MiniYaml.Merge(modData.Manifest.Cursors.Select(MiniYaml.FromFile)));
+			var sequenceYaml = MiniYaml.Merge(modData.Manifest.Cursors.Select(
+				s => MiniYaml.FromStream(modData.ModFiles.Open(s))));
 
 			var shadowIndex = new int[] { };
 
-			var nodesDict = sequences.ToDictionary();
+			var nodesDict = new MiniYaml(null, sequenceYaml).ToDictionary();
 			if (nodesDict.ContainsKey("ShadowIndex"))
 			{
 				Array.Resize(ref shadowIndex, shadowIndex.Length + 1);

OpenRA.Game/Graphics/SequenceProvider.cs

@@ -123,7 +123,10 @@ namespace OpenRA.Graphics
 
 		Sequences Load(List<MiniYamlNode> sequenceNodes)
 		{
-			var nodes = MiniYaml.Merge(modData.Manifest.Sequences.Select(MiniYaml.FromFile).Append(sequenceNodes));
+			var nodes = MiniYaml.Merge(modData.Manifest.Sequences
+				.Select(s => MiniYaml.FromStream(modData.ModFiles.Open(s)))
+				.Append(sequenceNodes));
+
 			var items = new Dictionary<string, UnitSequences>();
 			foreach (var n in nodes)
 			{

OpenRA.Game/Graphics/VoxelProvider.cs

@@ -19,11 +19,13 @@ namespace OpenRA.Graphics
 	{
 		static Dictionary<string, Dictionary<string, Voxel>> units;
 
-		public static void Initialize(string[] voxelFiles, List<MiniYamlNode> voxelNodes)
+		public static void Initialize(ModData modData, string[] voxelFiles, List<MiniYamlNode> voxelNodes)
 		{
 			units = new Dictionary<string, Dictionary<string, Voxel>>();
 
-			var sequences = MiniYaml.Merge(voxelFiles.Select(MiniYaml.FromFile));
+			var sequences = MiniYaml.Merge(voxelFiles.Select(
+				s => MiniYaml.FromStream(modData.ModFiles.Open(s))));
+
 			foreach (var s in sequences)
 				LoadVoxelsForUnit(s.Key, s.Value);
 

OpenRA.Game/Map/TileSet.cs

@@ -193,7 +193,7 @@ namespace OpenRA
 
 		public TileSet(ModData modData, string filepath)
 		{
-			var yaml = MiniYaml.DictFromFile(filepath);
+			var yaml = MiniYaml.DictFromStream(modData.ModFiles.Open(filepath));
 
 			// General info
 			FieldLoader.Load(this, yaml["General"]);

OpenRA.Game/ModData.cs

@@ -56,6 +56,8 @@ namespace OpenRA
 				LoadScreen.Display();
 			}
 
+			ModFiles.LoadFromManifest(Manifest);
+
 			WidgetLoader = new WidgetLoader(this);
 			RulesetCache = new RulesetCache(this);
 			RulesetCache.LoadingProgress += HandleLoadingProgress;
@@ -86,17 +88,12 @@ namespace OpenRA
 				LoadScreen.Display();
 		}
 
-		public void MountFiles()
-		{
-			ModFiles.LoadFromManifest(Manifest);
-		}
-
 		public void InitializeLoaders()
 		{
 			// all this manipulation of static crap here is nasty and breaks
 			// horribly when you use ModData in unexpected ways.
-			ChromeMetrics.Initialize(Manifest.ChromeMetrics);
-			ChromeProvider.Initialize(Manifest.Chrome);
+			ChromeMetrics.Initialize(this);
+			ChromeProvider.Initialize(this);
 
 			if (VoxelLoader != null)
 				VoxelLoader.Dispose();
@@ -133,7 +130,9 @@ namespace OpenRA
 				return;
 			}
 
-			var yaml = MiniYaml.Merge(Manifest.Translations.Select(MiniYaml.FromFile).Append(map.TranslationDefinitions));
+			var yaml = MiniYaml.Merge(Manifest.Translations
+				.Select(t => MiniYaml.FromStream(ModFiles.Open(t)))
+				.Append(map.TranslationDefinitions));
 			Languages = yaml.Select(t => t.Key).ToArray();
 
 			foreach (var y in yaml)
@@ -188,7 +187,7 @@ namespace OpenRA
 				foreach (var entry in map.Rules.Music)
 					entry.Value.Load();
 
-			VoxelProvider.Initialize(Manifest.VoxelSequences, map.VoxelSequenceDefinitions);
+			VoxelProvider.Initialize(this, Manifest.VoxelSequences, map.VoxelSequenceDefinitions);
 			VoxelLoader.Finish();
 
 			return map;

OpenRA.Game/Scripting/ScriptContext.cs

@@ -11,6 +11,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Reflection;
 using Eluant;
@@ -158,7 +159,7 @@ namespace OpenRA.Scripting
 				.ToArray();
 
 			runtime.Globals["GameDir"] = Platform.GameDir;
-			runtime.DoBuffer(Game.ModData.ModFiles.Open(Platform.ResolvePath(".", "lua", "scriptwrapper.lua")).ReadAllText(), "scriptwrapper.lua").Dispose();
+			runtime.DoBuffer(File.Open(Platform.ResolvePath(".", "lua", "scriptwrapper.lua"), FileMode.Open).ReadAllText(), "scriptwrapper.lua").Dispose();
 			tick = (LuaFunction)runtime.Globals["Tick"];
 
 			// Register globals

OpenRA.Game/Widgets/ChromeMetrics.cs

@@ -17,11 +17,11 @@ namespace OpenRA.Widgets
 	{
 		static Dictionary<string, string> data = new Dictionary<string, string>();
 
-		public static void Initialize(IEnumerable<string> yaml)
+		public static void Initialize(ModData modData)
 		{
 			data = new Dictionary<string, string>();
-
-			var metrics = MiniYaml.Merge(yaml.Select(MiniYaml.FromFile));
+			var metrics = MiniYaml.Merge(modData.Manifest.ChromeMetrics.Select(
+				y => MiniYaml.FromStream(modData.ModFiles.Open(y))));
 			foreach (var m in metrics)
 				foreach (var n in m.Value.Nodes)
 					data[n.Key] = n.Value.Value;

OpenRA.Game/Widgets/RootWidget.cs

@@ -27,7 +27,7 @@ namespace OpenRA.Widgets
 
 				if (hk == Game.Settings.Keys.DevReloadChromeKey)
 				{
-					ChromeProvider.Initialize(Game.ModData.Manifest.Chrome);
+					ChromeProvider.Initialize(Game.ModData);
 					return true;
 				}
 

OpenRA.Game/Widgets/WidgetLoader.cs

@@ -24,7 +24,7 @@ namespace OpenRA
 		{
 			this.modData = modData;
 
-			foreach (var file in modData.Manifest.ChromeLayout.Select(a => MiniYaml.FromFile(a)))
+			foreach (var file in modData.Manifest.ChromeLayout.Select(a => MiniYaml.FromStream(modData.ModFiles.Open(a))))
 				foreach (var w in file)
 				{
 					var key = w.Key.Substring(w.Key.IndexOf('@') + 1);

OpenRA.Mods.Common/UtilityCommands/ConvertSpriteToPngCommand.cs

@@ -33,7 +33,6 @@ namespace OpenRA.Mods.Common.UtilityCommands
 		{
 			// HACK: The engine code assumes that Game.modData is set.
 			Game.ModData = modData;
-			Game.ModData.MountFiles();
 
 			var src = args[1];
 			var shadowIndex = new int[] { };

OpenRA.Mods.Common/UtilityCommands/GenerateMinimapCommand.cs

@@ -27,7 +27,6 @@ namespace OpenRA.Mods.Common.UtilityCommands
 		public void Run(ModData modData, string[] args)
 		{
 			Game.ModData = modData;
-			modData.MountFiles();
 
 			var map = new Map(args[1]);
 			var minimap = Minimap.RenderMapPreview(map.Rules.TileSets[map.Tileset], map, true);

OpenRA.Mods.Common/UtilityCommands/ImportLegacyMapCommand.cs

@@ -44,7 +44,6 @@ namespace OpenRA.Mods.Common.UtilityCommands
 		{
 			// HACK: The engine code assumes that Game.modData is set.
 			Game.ModData = modData;
-			Game.ModData.MountFiles();
 
 			Rules = Game.ModData.RulesetCache.Load();
 

OpenRA.Mods.Common/Widgets/Logic/MissionBrowserLogic.cs

@@ -95,7 +95,9 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 			// Add a group for each campaign
 			if (modData.Manifest.Missions.Any())
 			{
-				var yaml = MiniYaml.Merge(modData.Manifest.Missions.Select(MiniYaml.FromFile));
+				var yaml = MiniYaml.Merge(modData.Manifest.Missions.Select(
+					m => MiniYaml.FromStream(modData.ModFiles.Open(m))));
+
 				foreach (var kv in yaml)
 				{
 					var missionMapPaths = kv.Value.Nodes.Select(n => Path.GetFullPath(n.Key)).ToList();

OpenRA.Mods.TS/UtilityCommands/ImportTSMapCommand.cs

@@ -154,7 +154,6 @@ namespace OpenRA.Mods.TS.UtilityCommands
 		{
 			// HACK: The engine code assumes that Game.modData is set.
 			Game.ModData = modData;
-			Game.ModData.MountFiles();
 
 			var filename = args[1];
 			var file = new IniFile(File.Open(args[1], FileMode.Open));

OpenRA.Mods.TS/UtilityCommands/LegacyTilesetImporter.cs

@@ -29,7 +29,6 @@ namespace OpenRA.Mods.TS.UtilityCommands
 		{
 			// HACK: The engine code assumes that Game.modData is set.
 			Game.ModData = modData;
-			Game.ModData.MountFiles();
 
 			var file = new IniFile(File.Open(args[1], FileMode.Open));
 			var extension = args[2];

thirdparty/fetch-travis-assets.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# Die on any error for Travis CI to automatically retry:
+set -e
+if [ ! -e ./Support/Content/ra/conquer.mix ]; then
+	echo "Downloading RA mod content"
+	mkdir -p ./Support/Content/ra/
+	cd ./Support/Content/ra/
+	curl -s -L -O `curl -s -L http://www.openra.net/packages/ra-mirrors.txt | head -n1`
+	unzip ra-packages.zip
+	rm ra-packages.zip
+	cd ../../../
+fi
+
+if [ ! -e ./Support/Content/cnc/conquer.mix ]; then
+	echo "Downloading TD mod content"
+	mkdir -p ./Support/Content/cnc/
+	cd ./Support/Content/cnc/
+	curl -s -L -O `curl -s -L http://www.openra.net/packages/cnc-mirrors.txt | head -n1`
+	unzip cnc-packages.zip
+	rm cnc-packages.zip
+	cd ../../../
+fi
+
+if [ ! -e ./Support/Content/d2k/DATA.R8 ]; then
+	echo "Downloading D2K mod content"
+	mkdir -p ./Support/Content/d2k/
+	cd ./Support/Content/d2k/
+	curl -s -L -O `curl -s -L http://www.openra.net/packages/d2k-103-mirrors.txt | head -n1`
+	unzip d2k-103-packages.zip
+	rm d2k-103-packages.zip
+	cd ../../../
+fi
+
+if [ ! -e ./Support/Content/ts/conquer.mix ]; then
+	echo "Downloading TS mod content"
+	mkdir -p ./Support/Content/ts/
+	cd ./Support/Content/ts/
+	curl -s -L -O `curl -s -L http://www.openra.net/packages/ts-mirrors.txt | head -n1`
+	unzip ts-packages.zip
+	rm ts-packages.zip
+	cd ../../../
+fi