Switch GeoIP database from MaxMind to IP2Location.

The IP2Location data is licensed under CC BY-SA, which
allows us to distribute the database with releases.

.gitignore

@@ -26,7 +26,7 @@ mods/*/*.pdb
 /*.exe
 /*.exe.config
 thirdparty/download/*
-*.mmdb.gz
+IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP
 
 # backup files by various editors
 *~

AUTHORS

@@ -160,9 +160,6 @@ FreeType License.
 
 Using OpenAL Soft distributed under the GNU LGPL.
 
-Using MaxMind GeoIP2 .NET API distributed under
-the Apache 2.0 license.
-
 Using SDL2-CS and OpenAL-CS created by Ethan
 Lee and released under the zlib license.
 
@@ -182,6 +179,9 @@ Krueger and distributed under the GNU GPL terms.
 Using rix0rrr.BeaconLib developed by Rico Huijbers
 distributed under MIT License.
 
+This site or product includes IP2Location LITE data
+available from http://www.ip2location.com.
+
 Finally, special thanks goes to the original teams
 at Westwood Studios and EA for creating the classic
 games which OpenRA aims to reimagine.

Makefile

@@ -40,7 +40,7 @@
 WHITELISTED_OPENRA_ASSEMBLIES = OpenRA.Game.exe OpenRA.Utility.exe OpenRA.Platforms.Default.dll OpenRA.Mods.Common.dll OpenRA.Mods.Cnc.dll OpenRA.Mods.D2k.dll OpenRA.Game.dll
 
 # These are explicitly shipped alongside our core files by the packaging script
-WHITELISTED_THIRDPARTY_ASSEMBLIES = ICSharpCode.SharpZipLib.dll FuzzyLogicLibrary.dll MaxMind.Db.dll Eluant.dll rix0rrr.BeaconLib.dll Open.Nat.dll SDL2-CS.dll OpenAL-CS.dll 
+WHITELISTED_THIRDPARTY_ASSEMBLIES = ICSharpCode.SharpZipLib.dll FuzzyLogicLibrary.dll Eluant.dll rix0rrr.BeaconLib.dll Open.Nat.dll SDL2-CS.dll OpenAL-CS.dll
 
 # These are shipped in our custom minimal mono runtime and also available in the full system-installed .NET/mono stack
 # This list *must* be kept in sync with the files packaged by the AppImageSupport and OpenRALauncherOSX repositories
@@ -157,10 +157,11 @@ ifeq ($(WIN32), $(filter $(WIN32),true yes y on 1))
 else
 	@$(MSBUILD) -t:build -p:Configuration=Release
 endif
+	@./fetch-geoip.sh
 
 clean:
 	@ $(MSBUILD) -t:clean
-	@-$(RM_F) *.config
+	@-$(RM_F) *.config IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP
 	@-$(RM_F) *.exe *.dll *.dylib ./OpenRA*/*.dll *.pdb mods/**/*.dll mods/**/*.pdb *.resources
 	@-$(RM_RF) ./*/bin ./*/obj
 	@-$(RM_RF) ./thirdparty/download
@@ -218,6 +219,7 @@ install-engine:
 	@$(INSTALL_DATA) VERSION "$(DATA_INSTALL_DIR)/VERSION"
 	@$(INSTALL_DATA) AUTHORS "$(DATA_INSTALL_DIR)/AUTHORS"
 	@$(INSTALL_DATA) COPYING "$(DATA_INSTALL_DIR)/COPYING"
+	@$(INSTALL_DATA) IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP "$(DATA_INSTALL_DIR)/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP"
 
 	@$(CP_R) glsl "$(DATA_INSTALL_DIR)"
 	@$(CP_R) lua "$(DATA_INSTALL_DIR)"
@@ -227,7 +229,6 @@ install-engine:
 	@$(INSTALL_PROGRAM) ICSharpCode.SharpZipLib.dll "$(DATA_INSTALL_DIR)"
 	@$(INSTALL_PROGRAM) FuzzyLogicLibrary.dll "$(DATA_INSTALL_DIR)"
 	@$(INSTALL_PROGRAM) Open.Nat.dll "$(DATA_INSTALL_DIR)"
-	@$(INSTALL_PROGRAM) MaxMind.Db.dll "$(DATA_INSTALL_DIR)"
 	@$(INSTALL_PROGRAM) rix0rrr.BeaconLib.dll "$(DATA_INSTALL_DIR)"
 
 install-common-mod-files:

OpenRA.Game/Network/GeoIP.cs

@@ -11,63 +11,119 @@
 
 using System;
 using System.IO;
+using System.Linq;
 using System.Net;
-using ICSharpCode.SharpZipLib.GZip;
-using MaxMind.Db;
+using System.Net.Sockets;
+using System.Numerics;
+using ICSharpCode.SharpZipLib.Zip;
 
 namespace OpenRA.Network
 {
 	public class GeoIP
 	{
-		public class GeoIP2Record
+		class IP2LocationReader
 		{
-			[Constructor]
-			public GeoIP2Record(GeoIP2Country country)
+			public readonly DateTime Date;
+			readonly Stream stream;
+			readonly uint columnCount;
+			readonly uint v4Count;
+			readonly uint v4Offset;
+			readonly uint v6Count;
+			readonly uint v6Offset;
+
+			public IP2LocationReader(Stream source)
 			{
-				Country = country;
+				// Copy stream data for reuse
+				stream = new MemoryStream();
+				source.CopyTo(stream);
+				stream.Position = 0;
+
+				if (stream.ReadUInt8() != 1)
+					throw new InvalidDataException("Only IP2Location type 1 databases are supported.");
+
+				columnCount = stream.ReadUInt8();
+				var year = stream.ReadUInt8();
+				var month = stream.ReadUInt8();
+				var day = stream.ReadUInt8();
+				Date = new DateTime(2000 + year, month, day);
+
+				v4Count = stream.ReadUInt32();
+				v4Offset = stream.ReadUInt32();
+				v6Count = stream.ReadUInt32();
+				v6Offset = stream.ReadUInt32();
 			}
 
-			public GeoIP2Country Country { get; set; }
+			BigInteger AddressForIndex(long index, bool isIPv6)
+			{
+				var start = isIPv6 ? v6Offset : v4Offset;
+				var offset = isIPv6 ? 12 : 0;
+				stream.Seek(start + index * (4 * columnCount + offset) - 1, SeekOrigin.Begin);
+				return new BigInteger(stream.ReadBytes(isIPv6 ? 16 : 4).Append((byte)0).ToArray());
 			}
 
-		public class GeoIP2Country
+			string CountryForIndex(long index, bool isIPv6)
 			{
-			[Constructor]
-			public GeoIP2Country(GeoIP2CountryNames names)
-			{
-				Names = names;
+				// Read file offset for country entry
+				var start = isIPv6 ? v6Offset : v4Offset;
+				var offset = isIPv6 ? 12 : 0;
+				stream.Seek(start + index * (4 * columnCount + offset) + offset + 3, SeekOrigin.Begin);
+				var countryOffset = stream.ReadUInt32();
+
+				// Read length-prefixed country name
+				stream.Seek(countryOffset + 3, SeekOrigin.Begin);
+				var length = stream.ReadUInt8();
+
+				// "-" is used to represent an unknown country in the database
+				var country = stream.ReadASCII(length);
+				return country != "-" ? country : null;
 			}
 
-			public GeoIP2CountryNames Names { get; set; }
+			public string LookupCountry(IPAddress ip)
+			{
+				var isIPv6 = ip.AddressFamily == AddressFamily.InterNetworkV6;
+				if (!isIPv6 && ip.AddressFamily != AddressFamily.InterNetwork)
+					return null;
+
+				// Locate IP using a binary search
+				// The IP2Location python parser has an additional
+				// optimization that can jump directly to the row, but this adds
+				// extra complexity that isn't obviously needed for our limited database size
+				long low = 0;
+				long high = isIPv6 ? v6Count : v4Count;
+
+				// Append an empty byte to force the data to be treated as unsigned
+				var ipNumber = new BigInteger(ip.GetAddressBytes().Reverse().Append((byte)0).ToArray());
+				while (low <= high)
+				{
+					var mid = (low + high) / 2;
+					var min = AddressForIndex(mid, isIPv6);
+					var max = AddressForIndex(mid + 1, isIPv6);
+					if (min <= ipNumber && ipNumber < max)
+						return CountryForIndex(mid, isIPv6);
+
+					if (ipNumber < min)
+						high = mid - 1;
+					else
+						low = mid + 1;
 				}
 
-		public class GeoIP2CountryNames
-		{
-			[Constructor]
-			public GeoIP2CountryNames(string en)
-			{
-				English = en;
+				return null;
+			}
 		}
 
-			public string English { get; set; }
-		}
+		static IP2LocationReader database;
 
-		static Reader database;
-
-		public static void Initialize(string databasePath)
+		public static void Initialize(string databasePath = "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP")
 		{
-			if (string.IsNullOrEmpty(databasePath))
+			if (!File.Exists(databasePath))
 				return;
 
 			try
 			{
-				using (var fileStream = new FileStream(databasePath, FileMode.Open, FileAccess.Read))
+				using (var z = new ZipFile(databasePath))
 				{
-					if (databasePath.EndsWith(".gz"))
-						using (var gzipStream = new GZipInputStream(fileStream))
-							database = new Reader(gzipStream);
-					else
-						database = new Reader(fileStream);
+					var entry = z.FindEntry("IP2LOCATION-LITE-DB1.IPV6.BIN", false);
+					database = new IP2LocationReader(z.GetInputStream(entry));
 				}
 			}
 			catch (Exception e)
@@ -82,9 +138,7 @@ namespace OpenRA.Network
 			{
 				try
 				{
-					var record = database.Find<GeoIP2Record>(ip);
-					if (record != null)
-						return record.Country.Names.English;
+					return database.LookupCountry(ip);
 				}
 				catch (Exception e)
 				{

OpenRA.Game/OpenRA.Game.csproj

@@ -58,10 +58,6 @@
       <HintPath>..\thirdparty\download\ICSharpCode.SharpZipLib.dll</HintPath>
       <Private>False</Private>
     </Reference>
-    <Reference Include="MaxMind.Db">
-      <HintPath>..\thirdparty\download\MaxMind.Db.dll</HintPath>
-      <Private>False</Private>
-    </Reference>
     <ProjectReference Include="..\OpenRA.PostProcess\OpenRA.PostProcess.csproj">
       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
     </ProjectReference>

OpenRA.Game/Server/Server.cs

@@ -149,7 +149,8 @@ namespace OpenRA.Server
 
 			randomSeed = (int)DateTime.Now.ToBinary();
 
-			GeoIP.Initialize(settings.GeoIPDatabase);
+			if (type != ServerType.Local && settings.EnableGeoIP)
+				GeoIP.Initialize();
 
 			if (UPnP.Status == UPnPStatus.Enabled)
 				UPnP.ForwardPort(Settings.ListenPort, Settings.ListenPort).Wait();

OpenRA.Game/Settings.cs

@@ -82,13 +82,12 @@ namespace OpenRA
 		[Desc("Sets the timestamp format. Defaults to the ISO 8601 standard.")]
 		public string TimestampFormat = "yyyy-MM-ddTHH:mm:ss";
 
-		[Desc("Path to a MaxMind GeoLite2 database to use for player geo-location.",
-			"Database files can be downloaded from https://dev.maxmind.com/geoip/geoip2/geolite2/")]
-		public string GeoIPDatabase = null;
-
 		[Desc("Allow clients to see anonymised IPs for other clients.")]
 		public bool ShareAnonymizedIPs = true;
 
+		[Desc("Allow clients to see the country of other clients.")]
+		public bool EnableGeoIP = true;
+
 		public ServerSettings Clone()
 		{
 			return (ServerSettings)MemberwiseClone();

appveyor.yml

@@ -51,6 +51,7 @@ test_script:
   - nunit3-console OpenRA.Test.dll --result=myresults.xml;format=AppVeyor
 
 after_test:
+  - appveyor DownloadFile "https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP" -FileName IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP
   - appveyor DownloadFile "https://raw.githubusercontent.com/wiki/OpenRA/OpenRA/Changelog.md" -FileName Changelog.md
   - make docs
   - ps: dir *.md | % {gc $_ -Raw | .\ConvertFrom-Markdown.ps1 | Out-File -FilePath "$($_.Name.TrimEnd(".md")).html"}

fetch-geoip.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Download the IP2Location country database for use by the game server
+
+####
+# This file must stay /bin/sh and POSIX compliant for macOS and BSD portability.
+# Copy-paste the entire script into http://shellcheck.net to check.
+####
+
+# Set the working directory to the location of this script
+cd "$(dirname "$0")" || exit 1
+
+# Database does not exist or is older than 30 days.
+if [ -z "$(find . -path ./IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP -mtime -30 -print)" ]; then
+	rm -f IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP
+	echo "Downloading IP2Location GeoIP database."
+	if command -v curl >/dev/null 2>&1; then
+		curl -s -L -O https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP || echo "Warning: Download failed"
+	else
+		wget -cq https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP || echo "Warning: Download failed"
+	fi
+fi

launch-dedicated.cmd

@@ -8,20 +8,19 @@ set ListenPort=1234
 set AdvertiseOnline=True
 set Password=""
 
-set GeoIPDatabase=""
-
 set RequireAuthentication=False
 set ProfileIDBlacklist=""
 set ProfileIDWhitelist=""
 
 set EnableSingleplayer=False
 set EnableSyncReports=False
+set EnableGeoIP=True
 set ShareAnonymizedIPs=True
 
 set SupportDir=""
 
 :loop
 
-OpenRA.Server.exe Game.Mod=%Mod% Server.Name=%Name% Server.ListenPort=%ListenPort% Server.AdvertiseOnline=%AdvertiseOnline% Server.EnableSingleplayer=%EnableSingleplayer% Server.Password=%Password% Server.GeoIPDatabase=%GeoIPDatabase% Server.RequireAuthentication=%RequireAuthentication% Server.ProfileIDBlacklist=%ProfileIDBlacklist% Server.ProfileIDWhitelist=%ProfileIDWhitelist% Server.EnableSyncReports=%EnableSyncReports% Server.ShareAnonymizedIPs=%ShareAnonymizedIPs% Engine.SupportDir=%SupportDir%
+OpenRA.Server.exe Game.Mod=%Mod% Server.Name=%Name% Server.ListenPort=%ListenPort% Server.AdvertiseOnline=%AdvertiseOnline% Server.EnableSingleplayer=%EnableSingleplayer% Server.Password=%Password% Server.RequireAuthentication=%RequireAuthentication% Server.ProfileIDBlacklist=%ProfileIDBlacklist% Server.ProfileIDWhitelist=%ProfileIDWhitelist% Server.EnableSyncReports=%EnableSyncReports% Server.EnableGeoIP=%EnableGeoIP% Server.ShareAnonymizedIPs=%ShareAnonymizedIPs% Engine.SupportDir=%SupportDir%
 
 goto loop

launch-dedicated.sh

@@ -12,14 +12,13 @@ ListenPort="${ListenPort:-"1234"}"
 AdvertiseOnline="${AdvertiseOnline:-"True"}"
 Password="${Password:-""}"
 
-GeoIPDatabase="${GeoIPDatabase:-""}"
-
 RequireAuthentication="${RequireAuthentication:-"False"}"
 ProfileIDBlacklist="${ProfileIDBlacklist:-""}"
 ProfileIDWhitelist="${ProfileIDWhitelist:-""}"
 
 EnableSingleplayer="${EnableSingleplayer:-"False"}"
 EnableSyncReports="${EnableSyncReports:-"False"}"
+EnableGeoIP="${EnableGeoIP:-"True"}"
 ShareAnonymizedIPs="${ShareAnonymizedIPs:-"True"}"
 
 SupportDir="${SupportDir:-""}"
@@ -36,6 +35,7 @@ while true; do
      Server.ProfileIDBlacklist="$ProfileIDBlacklist" \
      Server.ProfileIDWhitelist="$ProfileIDWhitelist" \
      Server.EnableSyncReports="$EnableSyncReports" \
+     Server.EnableGeoIP="$EnableGeoIP" \
      Server.ShareAnonymizedIPs="$ShareAnonymizedIPs" \
      Engine.SupportDir="$SupportDir"
 done

make.ps1

@@ -21,6 +21,13 @@ function All-Command
 	{
 		Write-Host "Build succeeded." -ForegroundColor Green
 	}
+
+	if (!(Test-Path "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP") -Or (((get-date) - (get-item "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP").LastWriteTime) -gt (new-timespan -days 30)))
+	{
+		echo "Downloading IP2Location GeoIP database."
+		$target = Join-Path $pwd.ToString() "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP"
+		(New-Object System.Net.WebClient).DownloadFile("https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP", $target)
+	}
 }
 
 function Clean-Command 

packaging/windows/OpenRA.nsi

@@ -131,7 +131,7 @@ Section "Game" GAME
 	File "${SRCDIR}\SDL2-CS.dll"
 	File "${SRCDIR}\OpenAL-CS.dll"
 	File "${SRCDIR}\global mix database.dat"
-	File "${SRCDIR}\MaxMind.Db.dll"
+	File "${SRCDIR}\IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP"
 	File "${SRCDIR}\eluant.dll"
 	File "${SRCDIR}\rix0rrr.BeaconLib.dll"
 	File "${DEPSDIR}\soft_oal.dll"
@@ -248,8 +248,7 @@ Function ${UN}Clean
 	Delete $INSTDIR\TiberianDawn.ico
 	Delete $INSTDIR\Dune2000.ico
 	Delete "$INSTDIR\global mix database.dat"
-	Delete $INSTDIR\MaxMind.Db.dll
-	Delete $INSTDIR\KopiLua.dll
+	Delete $INSTDIR\IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP
 	Delete $INSTDIR\soft_oal.dll
 	Delete $INSTDIR\SDL2.dll
 	Delete $INSTDIR\lua51.dll

thirdparty/fetch-thirdparty-deps.ps1

@@ -19,14 +19,6 @@ if (!(Test-Path "ICSharpCode.SharpZipLib.dll"))
 	rmdir SharpZipLib -Recurse
 }
 
-if (!(Test-Path "MaxMind.Db.dll"))
-{
-	echo "Fetching MaxMind.Db from NuGet."
-	./nuget.exe install MaxMind.Db -Version 2.0.0 -ExcludeVersion -Verbosity quiet -Source nuget.org
-	cp MaxMind.Db/lib/net45/MaxMind.Db.* .
-	rmdir MaxMind.Db -Recurse
-}
-
 if (!(Test-Path "nunit.framework.dll"))
 {
 	echo "Fetching NUnit from NuGet."

thirdparty/fetch-thirdparty-deps.sh

@@ -20,13 +20,6 @@ if [ ! -f ICSharpCode.SharpZipLib.dll ]; then
 	rm -rf SharpZipLib
 fi
 
-if [ ! -f MaxMind.Db.dll ]; then
-	echo "Fetching MaxMind.Db from NuGet"
-	../noget.sh MaxMind.Db 2.0.0 -IgnoreDependencies
-	cp ./MaxMind.Db/lib/net45/MaxMind.Db.* .
-	rm -rf MaxMind.Db
-fi
-
 if [ ! -f nunit.framework.dll ]; then
 	echo "Fetching NUnit from NuGet"
 	../noget.sh NUnit 3.0.1