From 9c4faddc0fff9ac1133bce8375090caf21c93fb0 Mon Sep 17 00:00:00 2001 From: Paul Chote Date: Sun, 12 Apr 2020 14:36:35 +0100 Subject: [PATCH] Switch GeoIP database from MaxMind to IP2Location. The IP2Location data is licensed under CC BY-SA, which allows us to distribute the database with releases. --- .gitignore | 2 +- AUTHORS | 6 +- Makefile | 7 +- OpenRA.Game/Network/GeoIP.cs | 124 +++++++++++++++++++-------- OpenRA.Game/OpenRA.Game.csproj | 4 - OpenRA.Game/Server/Server.cs | 3 +- OpenRA.Game/Settings.cs | 7 +- appveyor.yml | 1 + fetch-geoip.sh | 21 +++++ launch-dedicated.cmd | 5 +- launch-dedicated.sh | 4 +- make.ps1 | 7 ++ packaging/windows/OpenRA.nsi | 5 +- thirdparty/fetch-thirdparty-deps.ps1 | 8 -- thirdparty/fetch-thirdparty-deps.sh | 7 -- 15 files changed, 137 insertions(+), 74 deletions(-) create mode 100755 fetch-geoip.sh diff --git a/.gitignore b/.gitignore index 2ec502db42..6c35410ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ mods/*/*.pdb /*.exe /*.exe.config thirdparty/download/* -*.mmdb.gz +IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP # backup files by various editors *~ diff --git a/AUTHORS b/AUTHORS index 44c75afe26..723ecb8a09 100644 --- a/AUTHORS +++ b/AUTHORS @@ -160,9 +160,6 @@ FreeType License. Using OpenAL Soft distributed under the GNU LGPL. -Using MaxMind GeoIP2 .NET API distributed under -the Apache 2.0 license. - Using SDL2-CS and OpenAL-CS created by Ethan Lee and released under the zlib license. @@ -182,6 +179,9 @@ Krueger and distributed under the GNU GPL terms. Using rix0rrr.BeaconLib developed by Rico Huijbers distributed under MIT License. +This site or product includes IP2Location LITE data +available from http://www.ip2location.com. + Finally, special thanks goes to the original teams at Westwood Studios and EA for creating the classic games which OpenRA aims to reimagine. diff --git a/Makefile b/Makefile index e0fc028d73..df2bfd43bf 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ WHITELISTED_OPENRA_ASSEMBLIES = OpenRA.Game.exe OpenRA.Utility.exe OpenRA.Platforms.Default.dll OpenRA.Mods.Common.dll OpenRA.Mods.Cnc.dll OpenRA.Mods.D2k.dll OpenRA.Game.dll # These are explicitly shipped alongside our core files by the packaging script -WHITELISTED_THIRDPARTY_ASSEMBLIES = ICSharpCode.SharpZipLib.dll FuzzyLogicLibrary.dll MaxMind.Db.dll Eluant.dll rix0rrr.BeaconLib.dll Open.Nat.dll SDL2-CS.dll OpenAL-CS.dll +WHITELISTED_THIRDPARTY_ASSEMBLIES = ICSharpCode.SharpZipLib.dll FuzzyLogicLibrary.dll Eluant.dll rix0rrr.BeaconLib.dll Open.Nat.dll SDL2-CS.dll OpenAL-CS.dll # These are shipped in our custom minimal mono runtime and also available in the full system-installed .NET/mono stack # This list *must* be kept in sync with the files packaged by the AppImageSupport and OpenRALauncherOSX repositories @@ -157,10 +157,11 @@ ifeq ($(WIN32), $(filter $(WIN32),true yes y on 1)) else @$(MSBUILD) -t:build -p:Configuration=Release endif + @./fetch-geoip.sh clean: @ $(MSBUILD) -t:clean - @-$(RM_F) *.config + @-$(RM_F) *.config IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP @-$(RM_F) *.exe *.dll *.dylib ./OpenRA*/*.dll *.pdb mods/**/*.dll mods/**/*.pdb *.resources @-$(RM_RF) ./*/bin ./*/obj @-$(RM_RF) ./thirdparty/download @@ -218,6 +219,7 @@ install-engine: @$(INSTALL_DATA) VERSION "$(DATA_INSTALL_DIR)/VERSION" @$(INSTALL_DATA) AUTHORS "$(DATA_INSTALL_DIR)/AUTHORS" @$(INSTALL_DATA) COPYING "$(DATA_INSTALL_DIR)/COPYING" + @$(INSTALL_DATA) IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP "$(DATA_INSTALL_DIR)/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP" @$(CP_R) glsl "$(DATA_INSTALL_DIR)" @$(CP_R) lua "$(DATA_INSTALL_DIR)" @@ -227,7 +229,6 @@ install-engine: @$(INSTALL_PROGRAM) ICSharpCode.SharpZipLib.dll "$(DATA_INSTALL_DIR)" @$(INSTALL_PROGRAM) FuzzyLogicLibrary.dll "$(DATA_INSTALL_DIR)" @$(INSTALL_PROGRAM) Open.Nat.dll "$(DATA_INSTALL_DIR)" - @$(INSTALL_PROGRAM) MaxMind.Db.dll "$(DATA_INSTALL_DIR)" @$(INSTALL_PROGRAM) rix0rrr.BeaconLib.dll "$(DATA_INSTALL_DIR)" install-common-mod-files: diff --git a/OpenRA.Game/Network/GeoIP.cs b/OpenRA.Game/Network/GeoIP.cs index b8b70afa4b..4408318a9e 100644 --- a/OpenRA.Game/Network/GeoIP.cs +++ b/OpenRA.Game/Network/GeoIP.cs @@ -11,63 +11,119 @@ using System; using System.IO; +using System.Linq; using System.Net; -using ICSharpCode.SharpZipLib.GZip; -using MaxMind.Db; +using System.Net.Sockets; +using System.Numerics; +using ICSharpCode.SharpZipLib.Zip; namespace OpenRA.Network { public class GeoIP { - public class GeoIP2Record + class IP2LocationReader { - [Constructor] - public GeoIP2Record(GeoIP2Country country) + public readonly DateTime Date; + readonly Stream stream; + readonly uint columnCount; + readonly uint v4Count; + readonly uint v4Offset; + readonly uint v6Count; + readonly uint v6Offset; + + public IP2LocationReader(Stream source) { - Country = country; + // Copy stream data for reuse + stream = new MemoryStream(); + source.CopyTo(stream); + stream.Position = 0; + + if (stream.ReadUInt8() != 1) + throw new InvalidDataException("Only IP2Location type 1 databases are supported."); + + columnCount = stream.ReadUInt8(); + var year = stream.ReadUInt8(); + var month = stream.ReadUInt8(); + var day = stream.ReadUInt8(); + Date = new DateTime(2000 + year, month, day); + + v4Count = stream.ReadUInt32(); + v4Offset = stream.ReadUInt32(); + v6Count = stream.ReadUInt32(); + v6Offset = stream.ReadUInt32(); } - public GeoIP2Country Country { get; set; } - } - - public class GeoIP2Country - { - [Constructor] - public GeoIP2Country(GeoIP2CountryNames names) + BigInteger AddressForIndex(long index, bool isIPv6) { - Names = names; + var start = isIPv6 ? v6Offset : v4Offset; + var offset = isIPv6 ? 12 : 0; + stream.Seek(start + index * (4 * columnCount + offset) - 1, SeekOrigin.Begin); + return new BigInteger(stream.ReadBytes(isIPv6 ? 16 : 4).Append((byte)0).ToArray()); } - public GeoIP2CountryNames Names { get; set; } - } - - public class GeoIP2CountryNames - { - [Constructor] - public GeoIP2CountryNames(string en) + string CountryForIndex(long index, bool isIPv6) { - English = en; + // Read file offset for country entry + var start = isIPv6 ? v6Offset : v4Offset; + var offset = isIPv6 ? 12 : 0; + stream.Seek(start + index * (4 * columnCount + offset) + offset + 3, SeekOrigin.Begin); + var countryOffset = stream.ReadUInt32(); + + // Read length-prefixed country name + stream.Seek(countryOffset + 3, SeekOrigin.Begin); + var length = stream.ReadUInt8(); + + // "-" is used to represent an unknown country in the database + var country = stream.ReadASCII(length); + return country != "-" ? country : null; } - public string English { get; set; } + public string LookupCountry(IPAddress ip) + { + var isIPv6 = ip.AddressFamily == AddressFamily.InterNetworkV6; + if (!isIPv6 && ip.AddressFamily != AddressFamily.InterNetwork) + return null; + + // Locate IP using a binary search + // The IP2Location python parser has an additional + // optimization that can jump directly to the row, but this adds + // extra complexity that isn't obviously needed for our limited database size + long low = 0; + long high = isIPv6 ? v6Count : v4Count; + + // Append an empty byte to force the data to be treated as unsigned + var ipNumber = new BigInteger(ip.GetAddressBytes().Reverse().Append((byte)0).ToArray()); + while (low <= high) + { + var mid = (low + high) / 2; + var min = AddressForIndex(mid, isIPv6); + var max = AddressForIndex(mid + 1, isIPv6); + if (min <= ipNumber && ipNumber < max) + return CountryForIndex(mid, isIPv6); + + if (ipNumber < min) + high = mid - 1; + else + low = mid + 1; + } + + return null; + } } - static Reader database; + static IP2LocationReader database; - public static void Initialize(string databasePath) + public static void Initialize(string databasePath = "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP") { - if (string.IsNullOrEmpty(databasePath)) + if (!File.Exists(databasePath)) return; try { - using (var fileStream = new FileStream(databasePath, FileMode.Open, FileAccess.Read)) + using (var z = new ZipFile(databasePath)) { - if (databasePath.EndsWith(".gz")) - using (var gzipStream = new GZipInputStream(fileStream)) - database = new Reader(gzipStream); - else - database = new Reader(fileStream); + var entry = z.FindEntry("IP2LOCATION-LITE-DB1.IPV6.BIN", false); + database = new IP2LocationReader(z.GetInputStream(entry)); } } catch (Exception e) @@ -82,9 +138,7 @@ namespace OpenRA.Network { try { - var record = database.Find(ip); - if (record != null) - return record.Country.Names.English; + return database.LookupCountry(ip); } catch (Exception e) { diff --git a/OpenRA.Game/OpenRA.Game.csproj b/OpenRA.Game/OpenRA.Game.csproj index b9baa27361..6e3e6c27d1 100644 --- a/OpenRA.Game/OpenRA.Game.csproj +++ b/OpenRA.Game/OpenRA.Game.csproj @@ -58,10 +58,6 @@ ..\thirdparty\download\ICSharpCode.SharpZipLib.dll False - - ..\thirdparty\download\MaxMind.Db.dll - False - false diff --git a/OpenRA.Game/Server/Server.cs b/OpenRA.Game/Server/Server.cs index dd334391d7..3ea9a8272f 100644 --- a/OpenRA.Game/Server/Server.cs +++ b/OpenRA.Game/Server/Server.cs @@ -149,7 +149,8 @@ namespace OpenRA.Server randomSeed = (int)DateTime.Now.ToBinary(); - GeoIP.Initialize(settings.GeoIPDatabase); + if (type != ServerType.Local && settings.EnableGeoIP) + GeoIP.Initialize(); if (UPnP.Status == UPnPStatus.Enabled) UPnP.ForwardPort(Settings.ListenPort, Settings.ListenPort).Wait(); diff --git a/OpenRA.Game/Settings.cs b/OpenRA.Game/Settings.cs index c284e68f39..32f4bcca1a 100644 --- a/OpenRA.Game/Settings.cs +++ b/OpenRA.Game/Settings.cs @@ -82,13 +82,12 @@ namespace OpenRA [Desc("Sets the timestamp format. Defaults to the ISO 8601 standard.")] public string TimestampFormat = "yyyy-MM-ddTHH:mm:ss"; - [Desc("Path to a MaxMind GeoLite2 database to use for player geo-location.", - "Database files can be downloaded from https://dev.maxmind.com/geoip/geoip2/geolite2/")] - public string GeoIPDatabase = null; - [Desc("Allow clients to see anonymised IPs for other clients.")] public bool ShareAnonymizedIPs = true; + [Desc("Allow clients to see the country of other clients.")] + public bool EnableGeoIP = true; + public ServerSettings Clone() { return (ServerSettings)MemberwiseClone(); diff --git a/appveyor.yml b/appveyor.yml index 7067091ec9..c2af614331 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,6 +51,7 @@ test_script: - nunit3-console OpenRA.Test.dll --result=myresults.xml;format=AppVeyor after_test: + - appveyor DownloadFile "https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP" -FileName IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP - appveyor DownloadFile "https://raw.githubusercontent.com/wiki/OpenRA/OpenRA/Changelog.md" -FileName Changelog.md - make docs - ps: dir *.md | % {gc $_ -Raw | .\ConvertFrom-Markdown.ps1 | Out-File -FilePath "$($_.Name.TrimEnd(".md")).html"} diff --git a/fetch-geoip.sh b/fetch-geoip.sh new file mode 100755 index 0000000000..964faf6b44 --- /dev/null +++ b/fetch-geoip.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Download the IP2Location country database for use by the game server + +#### +# This file must stay /bin/sh and POSIX compliant for macOS and BSD portability. +# Copy-paste the entire script into http://shellcheck.net to check. +#### + +# Set the working directory to the location of this script +cd "$(dirname "$0")" || exit 1 + +# Database does not exist or is older than 30 days. +if [ -z "$(find . -path ./IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP -mtime -30 -print)" ]; then + rm -f IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP + echo "Downloading IP2Location GeoIP database." + if command -v curl >/dev/null 2>&1; then + curl -s -L -O https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP || echo "Warning: Download failed" + else + wget -cq https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP || echo "Warning: Download failed" + fi +fi diff --git a/launch-dedicated.cmd b/launch-dedicated.cmd index b12d039c73..d9067012c3 100644 --- a/launch-dedicated.cmd +++ b/launch-dedicated.cmd @@ -8,20 +8,19 @@ set ListenPort=1234 set AdvertiseOnline=True set Password="" -set GeoIPDatabase="" - set RequireAuthentication=False set ProfileIDBlacklist="" set ProfileIDWhitelist="" set EnableSingleplayer=False set EnableSyncReports=False +set EnableGeoIP=True set ShareAnonymizedIPs=True set SupportDir="" :loop -OpenRA.Server.exe Game.Mod=%Mod% Server.Name=%Name% Server.ListenPort=%ListenPort% Server.AdvertiseOnline=%AdvertiseOnline% Server.EnableSingleplayer=%EnableSingleplayer% Server.Password=%Password% Server.GeoIPDatabase=%GeoIPDatabase% Server.RequireAuthentication=%RequireAuthentication% Server.ProfileIDBlacklist=%ProfileIDBlacklist% Server.ProfileIDWhitelist=%ProfileIDWhitelist% Server.EnableSyncReports=%EnableSyncReports% Server.ShareAnonymizedIPs=%ShareAnonymizedIPs% Engine.SupportDir=%SupportDir% +OpenRA.Server.exe Game.Mod=%Mod% Server.Name=%Name% Server.ListenPort=%ListenPort% Server.AdvertiseOnline=%AdvertiseOnline% Server.EnableSingleplayer=%EnableSingleplayer% Server.Password=%Password% Server.RequireAuthentication=%RequireAuthentication% Server.ProfileIDBlacklist=%ProfileIDBlacklist% Server.ProfileIDWhitelist=%ProfileIDWhitelist% Server.EnableSyncReports=%EnableSyncReports% Server.EnableGeoIP=%EnableGeoIP% Server.ShareAnonymizedIPs=%ShareAnonymizedIPs% Engine.SupportDir=%SupportDir% goto loop diff --git a/launch-dedicated.sh b/launch-dedicated.sh index 0ca89dd72f..31ef313d75 100755 --- a/launch-dedicated.sh +++ b/launch-dedicated.sh @@ -12,14 +12,13 @@ ListenPort="${ListenPort:-"1234"}" AdvertiseOnline="${AdvertiseOnline:-"True"}" Password="${Password:-""}" -GeoIPDatabase="${GeoIPDatabase:-""}" - RequireAuthentication="${RequireAuthentication:-"False"}" ProfileIDBlacklist="${ProfileIDBlacklist:-""}" ProfileIDWhitelist="${ProfileIDWhitelist:-""}" EnableSingleplayer="${EnableSingleplayer:-"False"}" EnableSyncReports="${EnableSyncReports:-"False"}" +EnableGeoIP="${EnableGeoIP:-"True"}" ShareAnonymizedIPs="${ShareAnonymizedIPs:-"True"}" SupportDir="${SupportDir:-""}" @@ -36,6 +35,7 @@ while true; do Server.ProfileIDBlacklist="$ProfileIDBlacklist" \ Server.ProfileIDWhitelist="$ProfileIDWhitelist" \ Server.EnableSyncReports="$EnableSyncReports" \ + Server.EnableGeoIP="$EnableGeoIP" \ Server.ShareAnonymizedIPs="$ShareAnonymizedIPs" \ Engine.SupportDir="$SupportDir" done diff --git a/make.ps1 b/make.ps1 index fc34a048bc..b99b46f5c7 100644 --- a/make.ps1 +++ b/make.ps1 @@ -21,6 +21,13 @@ function All-Command { Write-Host "Build succeeded." -ForegroundColor Green } + + if (!(Test-Path "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP") -Or (((get-date) - (get-item "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP").LastWriteTime) -gt (new-timespan -days 30))) + { + echo "Downloading IP2Location GeoIP database." + $target = Join-Path $pwd.ToString() "IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP" + (New-Object System.Net.WebClient).DownloadFile("https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP", $target) + } } function Clean-Command diff --git a/packaging/windows/OpenRA.nsi b/packaging/windows/OpenRA.nsi index 95026888a5..2dc3d21297 100644 --- a/packaging/windows/OpenRA.nsi +++ b/packaging/windows/OpenRA.nsi @@ -131,7 +131,7 @@ Section "Game" GAME File "${SRCDIR}\SDL2-CS.dll" File "${SRCDIR}\OpenAL-CS.dll" File "${SRCDIR}\global mix database.dat" - File "${SRCDIR}\MaxMind.Db.dll" + File "${SRCDIR}\IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP" File "${SRCDIR}\eluant.dll" File "${SRCDIR}\rix0rrr.BeaconLib.dll" File "${DEPSDIR}\soft_oal.dll" @@ -248,8 +248,7 @@ Function ${UN}Clean Delete $INSTDIR\TiberianDawn.ico Delete $INSTDIR\Dune2000.ico Delete "$INSTDIR\global mix database.dat" - Delete $INSTDIR\MaxMind.Db.dll - Delete $INSTDIR\KopiLua.dll + Delete $INSTDIR\IP2LOCATION-LITE-DB1.IPV6.BIN.ZIP Delete $INSTDIR\soft_oal.dll Delete $INSTDIR\SDL2.dll Delete $INSTDIR\lua51.dll diff --git a/thirdparty/fetch-thirdparty-deps.ps1 b/thirdparty/fetch-thirdparty-deps.ps1 index 50ba94fa19..6c2bd28462 100644 --- a/thirdparty/fetch-thirdparty-deps.ps1 +++ b/thirdparty/fetch-thirdparty-deps.ps1 @@ -19,14 +19,6 @@ if (!(Test-Path "ICSharpCode.SharpZipLib.dll")) rmdir SharpZipLib -Recurse } -if (!(Test-Path "MaxMind.Db.dll")) -{ - echo "Fetching MaxMind.Db from NuGet." - ./nuget.exe install MaxMind.Db -Version 2.0.0 -ExcludeVersion -Verbosity quiet -Source nuget.org - cp MaxMind.Db/lib/net45/MaxMind.Db.* . - rmdir MaxMind.Db -Recurse -} - if (!(Test-Path "nunit.framework.dll")) { echo "Fetching NUnit from NuGet." diff --git a/thirdparty/fetch-thirdparty-deps.sh b/thirdparty/fetch-thirdparty-deps.sh index c0e22d15b0..1252f557d2 100755 --- a/thirdparty/fetch-thirdparty-deps.sh +++ b/thirdparty/fetch-thirdparty-deps.sh @@ -20,13 +20,6 @@ if [ ! -f ICSharpCode.SharpZipLib.dll ]; then rm -rf SharpZipLib fi -if [ ! -f MaxMind.Db.dll ]; then - echo "Fetching MaxMind.Db from NuGet" - ../noget.sh MaxMind.Db 2.0.0 -IgnoreDependencies - cp ./MaxMind.Db/lib/net45/MaxMind.Db.* . - rm -rf MaxMind.Db -fi - if [ ! -f nunit.framework.dll ]; then echo "Fetching NUnit from NuGet" ../noget.sh NUnit 3.0.1