Merge pull request #8149 from Mailaender/fetch-geoip-db

Automatized GeoIP database download

.gitignore

@@ -29,6 +29,7 @@ mods/*/*.mdb
 /*.mdb
 /*.exe
 thirdparty/download/*
+*.mmdb.gz
 
 # backup files by various editors
 *~

Makefile

@@ -318,6 +318,7 @@ osx-dependencies: cli-dependencies
 	@ $(CP_R) thirdparty/download/osx/*.dll.config .
 
 dependencies: $(os-dependencies)
+	@./thirdparty/fetch-geoip-db.sh
 
 all-dependencies: cli-dependencies windows-dependencies osx-dependencies
 
@@ -353,7 +354,7 @@ install-core: default
 	@$(CP_R) mods/modchooser "$(DATA_INSTALL_DIR)/mods/"
 
 	@$(INSTALL_DATA) "global mix database.dat" "$(DATA_INSTALL_DIR)/global mix database.dat"
-	@$(INSTALL_DATA) "GeoLite2-Country.mmdb" "$(DATA_INSTALL_DIR)/GeoLite2-Country.mmdb"
+	@$(INSTALL_DATA) "GeoLite2-Country.mmdb.gz" "$(DATA_INSTALL_DIR)/GeoLite2-Country.mmdb.gz"
 	@$(INSTALL_DATA) AUTHORS "$(DATA_INSTALL_DIR)/AUTHORS"
 	@$(INSTALL_DATA) COPYING "$(DATA_INSTALL_DIR)/COPYING"
 

OpenRA.Game/Game.cs

@@ -15,7 +15,6 @@ using System.IO;
 using System.Linq;
 using System.Net;
 using System.Threading;
-using MaxMind.GeoIP2;
 using OpenRA.FileSystem;
 using OpenRA.Graphics;
 using OpenRA.Network;
@@ -44,8 +43,6 @@ namespace OpenRA
 		public static Renderer Renderer;
 		public static bool HasInputFocus = false;
 
-		public static DatabaseReader GeoIpDatabase;
-
 		public static OrderManager JoinServer(string host, int port, string password, bool recordReplay = true)
 		{
 			IConnection connection = new NetworkConnection(host, port);
@@ -216,14 +213,7 @@ namespace OpenRA
 				Settings.Server.AllowPortForward = false;
 			}
 
-			try
-			{
-				GeoIpDatabase = new DatabaseReader("GeoLite2-Country.mmdb");
-			}
-			catch (Exception e)
-			{
-				Log.Write("geoip", "DatabaseReader failed: {0}", e);
-			}
+			GeoIP.Initialize();
 
 			GlobalFileSystem.Mount(Platform.GameDir); // Needed to access shaders
 			var renderers = new[] { Settings.Graphics.Renderer, "Sdl2", null };

OpenRA.Game/Network/GeoIP.cs

@@ -0,0 +1,52 @@
+#region Copyright & License Information
+/*
+ * Copyright 2007-2015 The OpenRA Developers (see AUTHORS)
+ * This file is part of OpenRA, which is free software. It is made
+ * available to you under the terms of the GNU General Public License
+ * as published by the Free Software Foundation. For more information,
+ * see COPYING.
+ */
+#endregion
+
+using System;
+using System.IO;
+using ICSharpCode.SharpZipLib.Core;
+using ICSharpCode.SharpZipLib.GZip;
+using MaxMind.GeoIP2;
+
+namespace OpenRA.Network
+{
+	public class GeoIP
+	{
+		static DatabaseReader database;
+
+		public static void Initialize()
+		{
+			try
+			{
+				using (var fileStream = new FileStream("GeoLite2-Country.mmdb.gz", FileMode.Open, FileAccess.Read))
+					using (var gzipStream = new GZipInputStream(fileStream))
+						database = new DatabaseReader(gzipStream);
+			}
+			catch (Exception e)
+			{
+				Log.Write("geoip", "DatabaseReader failed: {0}", e);
+			}
+		}
+
+		public static string LookupCountry(string ip)
+		{
+			const string Unknown = "Unknown Location";
+
+			try
+			{
+				return database.Country(ip).Country.Name ?? Unknown;
+			}
+			catch (Exception e)
+			{
+				Log.Write("geoip", "LookupCountry failed: {0}", e);
+				return Unknown;
+			}
+		}
+	}
+}

OpenRA.Game/OpenRA.Game.csproj

@@ -128,6 +128,7 @@
     <Compile Include="Network\Connection.cs" />
     <Compile Include="Network\FrameData.cs" />
     <Compile Include="Network\GameServer.cs" />
+    <Compile Include="Network\GeoIP.cs" />
     <Compile Include="Network\Handshake.cs" />
     <Compile Include="Network\Order.cs" />
     <Compile Include="Network\OrderIO.cs" />

OpenRA.Mods.Common/Widgets/Logic/Lobby/ClientTooltipLogic.cs

@@ -85,7 +85,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 				address = UPnP.NatDevice.GetExternalIP().ToString();
 			var cachedDescriptiveIP = LobbyUtils.DescriptiveIpAddress(address);
 			ip.GetText = () => cachedDescriptiveIP;
-			var cachedCountryLookup = LobbyUtils.LookupCountry(address);
+			var cachedCountryLookup = GeoIP.LookupCountry(address);
 			location.GetText = () => cachedCountryLookup;
 		}
 	}

OpenRA.Mods.Common/Widgets/Logic/Lobby/LobbyUtils.cs

@@ -235,21 +235,6 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 			return ip;
 		}
 
-		public static string LookupCountry(string ip)
-		{
-			const string Unknown = "Unknown Location";
-
-			try
-			{
-				return Game.GeoIpDatabase.Country(ip).Country.Name ?? Unknown;
-			}
-			catch (Exception e)
-			{
-				Log.Write("geoip", "LookupCountry failed: {0}", e);
-				return Unknown;
-			}
-		}
-
 		public static void SetupClientWidget(Widget parent, Session.Slot s, Session.Client c, OrderManager orderManager, bool visible)
 		{
 			parent.Get("ADMIN_INDICATOR").IsVisible = () => c.IsAdmin;

OpenRA.Mods.Common/Widgets/Logic/ServerBrowserLogic.cs

@@ -252,7 +252,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 					var location = item.GetOrNull<LabelWidget>("LOCATION");
 					if (location != null)
 					{
-						var cachedServerLocation = LobbyUtils.LookupCountry(game.Address.Split(':')[0]);
+						var cachedServerLocation = GeoIP.LookupCountry(game.Address.Split(':')[0]);
 						location.GetText = () => cachedServerLocation;
 						location.GetColor = () => !compatible ? Color.DarkGray : !canJoin ? Color.LightGray : location.TextColor;
 					}

packaging/package-all.sh

@@ -35,7 +35,7 @@ FILES=('OpenRA.Game.exe' 'OpenRA.Game.exe.config' 'OpenRA.Editor.exe' 'OpenRA.Ut
 'OpenRA.Renderer.Sdl2.dll' 'OpenRA.Renderer.Null.dll' \
  'lua' 'glsl' 'mods/common' 'mods/ra' 'mods/cnc' 'mods/d2k' 'mods/modchooser' \
 'AUTHORS' 'COPYING' 'README.html' 'CONTRIBUTING.html' 'DOCUMENTATION.html' 'CHANGELOG.html' \
-'global mix database.dat' 'GeoLite2-Country.mmdb')
+'global mix database.dat' 'GeoLite2-Country.mmdb.gz')
 
 echo "Copying files..."
 for i in "${FILES[@]}"; do

packaging/windows/OpenRA.nsi

@@ -103,7 +103,7 @@ Section "Game" GAME
 	File "${SRCDIR}\MaxMind.GeoIP2.dll"
 	File "${SRCDIR}\Newtonsoft.Json.dll"
 	File "${SRCDIR}\RestSharp.dll"
-	File "${SRCDIR}\GeoLite2-Country.mmdb"
+	File "${SRCDIR}\GeoLite2-Country.mmdb.gz"
 	File "${SRCDIR}\eluant.dll"
 	File "${DEPSDIR}\soft_oal.dll"
 	File "${DEPSDIR}\SDL2.dll"
@@ -219,7 +219,7 @@ Function ${UN}Clean
 	Delete $INSTDIR\MaxMind.GeoIP2.dll
 	Delete $INSTDIR\Newtonsoft.Json.dll
 	Delete $INSTDIR\RestSharp.dll
-	Delete $INSTDIR\GeoLite2-Country.mmdb
+	Delete $INSTDIR\GeoLite2-Country.mmdb.gz
 	Delete $INSTDIR\KopiLua.dll
 	Delete $INSTDIR\soft_oal.dll
 	Delete $INSTDIR\SDL2.dll

thirdparty/fetch-geoip-db.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Die on any error for Travis CI to automatically retry:
+set -e
+
+download_dir="${0%/*}/download"
+mkdir -p "${download_dir}"
+cd "${download_dir}"
+
+filename="GeoLite2-Country.mmdb.gz"
+
+# Database does not exist or is older than 30 days.
+if [[ ! -e $filename ]] || [[ -n $(find . -name $filename -mtime +30 -print) ]]; then
+	rm -f $filename
+	echo "Updating GeoIP country database from MaxMind."
+	curl -s -L -O http://geolite.maxmind.com/download/geoip/database/$filename
+	cp $filename ../..
+fi
+

thirdparty/fetch-thirdparty-deps.ps1

@@ -130,4 +130,12 @@ if (!(Test-Path "Eluant.dll"))
 	(New-Object System.Net.WebClient).DownloadFile("https://github.com/OpenRA/Eluant/releases/download/20140425/Eluant.dll", $target)
 }
 
+if (!(Test-Path "GeoLite2-Country.mmdb.gz") -Or (((get-date) - (get-item "GeoLite2-Country.mmdb").LastWriteTime) -gt (new-timespan -days 30)))
+{
+	echo "Updating GeoIP country database from MaxMind."
+	$target = Join-Path $pwd.ToString() "GeoLite2-Country.mmdb.gz"
+	(New-Object System.Net.WebClient).DownloadFile("http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz", $target)
+	cp GeoLite2-Country.mmdb.gz ..\..
+}
+
 cd ..