Merge pull request #4849 from IvanSavenko/map_encoding

Better handling of encoding detection for maps and campaigns

client/renderSDL/CBitmapFont.cpp

@@ -99,9 +99,7 @@ static AtlasLayout doAtlasPacking(const std::map<int, Point> & images)
 void CBitmapFont::loadFont(const ResourcePath & resource, std::unordered_map<CodePoint, EntryFNT> & loadedChars)
 {
 	auto data = CResourceHandler::get()->load(resource)->readAll();
-	std::string modName = VLC->modh->findResourceOrigin(resource);
-	std::string modLanguage = VLC->modh->getModLanguage(modName);
-	std::string modEncoding = Languages::getLanguageOptions(modLanguage).encoding;
+	std::string modEncoding = VLC->modh->findResourceEncoding(resource);
 
 	height = data.first[5];
 

client/renderSDL/FontChain.cpp

@@ -60,11 +60,9 @@ bool FontChain::bitmapFontsPrioritized(const std::string & bitmapFontName) const
 	if (!vstd::isAlmostEqual(1.0, settings["video"]["fontScalingFactor"].Float()))
 		return false; // If player requested non-100% scaling - use scalable fonts
 
-	std::string modName = CGI->modh->findResourceOrigin(ResourcePath("data/" + bitmapFontName, EResType::BMP_FONT));
-	std::string fontLanguage = CGI->modh->getModLanguage(modName);
 	std::string gameLanguage = CGI->generaltexth->getPreferredLanguage();
-	std::string fontEncoding = Languages::getLanguageOptions(fontLanguage).encoding;
 	std::string gameEncoding = Languages::getLanguageOptions(gameLanguage).encoding;
+	std::string fontEncoding = CGI->modh->findResourceEncoding(ResourcePath("data/" + bitmapFontName, EResType::BMP_FONT));
 
 	// player uses language with different encoding than his bitmap fonts
 	// for example, Polish language with English fonts or Chinese language which can't use H3 fonts at all

lib/campaign/CampaignHandler.cpp

@@ -25,7 +25,6 @@
 #include "../modding/IdentifierStorage.h"
 #include "../modding/ModScope.h"
 #include "../texts/CGeneralTextHandler.h"
-#include "../texts/Languages.h"
 #include "../texts/TextOperations.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
@@ -64,8 +63,7 @@ std::unique_ptr<Campaign> CampaignHandler::getHeader( const std::string & name)
 {
 	ResourcePath resourceID(name, EResType::CAMPAIGN);
 	std::string modName = VLC->modh->findResourceOrigin(resourceID);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(resourceID);
 	
 	auto ret = std::make_unique<Campaign>();
 	auto fileStream = CResourceHandler::get(modName)->load(resourceID);
@@ -80,8 +78,7 @@ std::shared_ptr<CampaignState> CampaignHandler::getCampaign( const std::string &
 {
 	ResourcePath resourceID(name, EResType::CAMPAIGN);
 	std::string modName = VLC->modh->findResourceOrigin(resourceID);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(resourceID);
 	
 	auto ret = std::make_unique<CampaignState>();
 	

lib/mapping/CMapService.cpp

@@ -19,7 +19,6 @@
 #include "../modding/CModHandler.h"
 #include "../modding/ModScope.h"
 #include "../modding/CModInfo.h"
-#include "../texts/Languages.h"
 #include "../VCMI_Lib.h"
 
 #include "CMap.h"
@@ -34,8 +33,7 @@ VCMI_LIB_NAMESPACE_BEGIN
 std::unique_ptr<CMap> CMapService::loadMap(const ResourcePath & name, IGameCallback * cb) const
 {
 	std::string modName = VLC->modh->findResourceOrigin(name);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(name);
 
 	auto stream = getStreamFromFS(name);
 	return getMapLoader(stream, name.getName(), modName, encoding)->loadMap(cb);
@@ -44,8 +42,7 @@ std::unique_ptr<CMap> CMapService::loadMap(const ResourcePath & name, IGameCallb
 std::unique_ptr<CMapHeader> CMapService::loadMapHeader(const ResourcePath & name) const
 {
 	std::string modName = VLC->modh->findResourceOrigin(name);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(name);
 
 	auto stream = getStreamFromFS(name);
 	return getMapLoader(stream, name.getName(), modName, encoding)->loadMapHeader();

lib/modding/CModHandler.cpp

@@ -423,6 +423,36 @@ TModID CModHandler::findResourceOrigin(const ResourcePath & name) const
 	throw std::runtime_error("Resource with name " + name.getName() + " and type " + EResTypeHelper::getEResTypeAsString(name.getType()) + " wasn't found.");
 }
 
+std::string CModHandler::findResourceLanguage(const ResourcePath & name) const
+{
+	std::string modName = findResourceOrigin(name);
+	std::string modLanguage = getModLanguage(modName);
+	return modLanguage;
+}
+
+std::string CModHandler::findResourceEncoding(const ResourcePath & resource) const
+{
+	std::string modName = findResourceOrigin(resource);
+	std::string modLanguage = findResourceLanguage(resource);
+
+	bool potentiallyUserMadeContent = resource.getType() == EResType::MAP || resource.getType() == EResType::CAMPAIGN;
+	if (potentiallyUserMadeContent && modName == ModScope::scopeBuiltin() && modLanguage == "english")
+	{
+		// this might be a map or campaign that player downloaded manually and placed in Maps/ directory
+		// in this case, this file may be in user-preferred language, and not in same language as the rest of H3 data
+		// however at the moment we have no way to detect that for sure - file can be either in English or in user-preferred language
+		// but since all known H3 encodings (Win125X or GBK) are supersets of ASCII, we can safely load English data using encoding of user-preferred language
+		std::string preferredLanguage = VLC->generaltexth->getPreferredLanguage();
+		std::string fileEncoding = Languages::getLanguageOptions(modLanguage).encoding;
+		return fileEncoding;
+	}
+	else
+	{
+		std::string fileEncoding = Languages::getLanguageOptions(modLanguage).encoding;
+		return fileEncoding;
+	}
+}
+
 std::string CModHandler::getModLanguage(const TModID& modId) const
 {
 	if(modId == "core")

lib/modding/CModHandler.h

@@ -62,6 +62,12 @@ public:
 	/// returns ID of mod that provides selected file resource
 	TModID findResourceOrigin(const ResourcePath & name) const;
 
+	/// Returns assumed language ID of mod that provides selected file resource
+	std::string findResourceLanguage(const ResourcePath & name) const;
+
+	/// Returns assumed encoding of language of mod that provides selected file resource
+	std::string findResourceEncoding(const ResourcePath & name) const;
+
 	std::string getModLanguage(const TModID & modId) const;
 
 	std::set<TModID> getModDependencies(const TModID & modId) const;

lib/texts/CLegacyConfigParser.cpp

@@ -32,9 +32,7 @@ protected:
 CLegacyConfigParser::CLegacyConfigParser(const TextPath & resource)
 {
 	auto input = CResourceHandler::get()->load(resource);
-	std::string modName = VLC->modh->findResourceOrigin(resource);
-	std::string language = VLC->modh->getModLanguage(modName);
-	fileEncoding = Languages::getLanguageOptions(language).encoding;
+	fileEncoding = VLC->modh->findResourceEncoding(resource);
 
 	data.reset(new char[input->getSize()]);
 	input->read(reinterpret_cast<uint8_t*>(data.get()), input->getSize());