Unicode conversion functions now require source encoding

client/renderSDL/CBitmapFont.cpp

@@ -55,7 +55,7 @@ size_t CBitmapFont::getLineHeight() const
 
 size_t CBitmapFont::getGlyphWidth(const char * data) const
 {
-	std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])));
+	std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])), "ASCII");
 
 	if (localChar.size() == 1)
 	{
@@ -133,7 +133,7 @@ void CBitmapFont::renderText(SDL_Surface * surface, const std::string & data, co
 
 	for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
 	{
-		std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])));
+		std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])), "ASCII");
 
 		if (localChar.size() == 1)
 			renderCharacter(surface, chars[ui8(localChar[0])], color, posX, posY);

client/renderSDL/CBitmapHanFont.cpp

@@ -85,7 +85,7 @@ void CBitmapHanFont::renderText(SDL_Surface * surface, const std::string & data,
 
 	for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
 	{
-		std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])));
+		std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])), "GBK");
 
 		if (localChar.size() == 1)
 			fallback->renderCharacter(surface, fallback->chars[ui8(localChar[0])], color, posX, posY);
@@ -115,7 +115,7 @@ size_t CBitmapHanFont::getLineHeight() const
 
 size_t CBitmapHanFont::getGlyphWidth(const char * data) const
 {
-	std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])));
+	std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])), "GBK");
 
 	if (localChar.size() == 1)
 		return fallback->getGlyphWidth(data);

lib/CGeneralTextHandler.cpp

@@ -190,7 +190,7 @@ std::string CLegacyConfigParser::readString()
 	std::string str = readRawString();
 	if (TextOperations::isValidASCII(str))
 		return str;
-	return TextOperations::toUnicode(str);
+	return TextOperations::toUnicode(str, fileEncoding);
 }
 
 float CLegacyConfigParser::readNumber()

lib/CGeneralTextHandler.h

@@ -17,12 +17,12 @@ class JsonNode;
 /// Parser for any text files from H3
 class DLL_LINKAGE CLegacyConfigParser
 {
+	std::string fileEncoding;
+
 	std::unique_ptr<char[]> data;
 	char * curr;
 	char * end;
 
-	void init(const std::unique_ptr<CInputStream> & input);
-
 	/// extracts part of quoted string.
 	std::string extractQuotedPart();
 
@@ -56,6 +56,7 @@ public:
 	bool endLine();
 
 	explicit CLegacyConfigParser(std::string URI);
+private:
 	explicit CLegacyConfigParser(const std::unique_ptr<CInputStream> & input);
 };
 

lib/CModHandler.cpp

@@ -943,8 +943,8 @@ std::vector<std::string> CModHandler::getModList(std::string path)
 
 bool CModHandler::isScopeReserved(const TModID & scope)
 {
-	static const std::array<TModID, 3> reservedScopes = {
-		"core", "map", "game"
+	static const std::array<TModID, 6> reservedScopes = {
+		"core", "map", "game", "root", "saves", "config"
 	};
 
 	return std::find(reservedScopes.begin(), reservedScopes.end(), scope) != reservedScopes.end();

lib/TextOperations.cpp

@@ -112,21 +112,11 @@ bool TextOperations::isValidUnicodeString(const char * data, size_t size)
 	return true;
 }
 
-std::string TextOperations::toUnicode(const std::string &text)
-{
-	return toUnicode(text, CGeneralTextHandler::getInstalledEncoding());
-}
-
 std::string TextOperations::toUnicode(const std::string &text, const std::string &encoding)
 {
 	return boost::locale::conv::to_utf<char>(text, encoding);
 }
 
-std::string TextOperations::fromUnicode(const std::string & text)
-{
-	return fromUnicode(text, CGeneralTextHandler::getInstalledEncoding());
-}
-
 std::string TextOperations::fromUnicode(const std::string &text, const std::string &encoding)
 {
 	return boost::locale::conv::from_utf<char>(text, encoding);

lib/TextOperations.h

@@ -31,12 +31,10 @@ namespace TextOperations
 	bool DLL_LINKAGE isValidUnicodeString(const char * data, size_t size);
 
 	/// converts text to UTF-8 from specified encoding or from one specified in settings
-	std::string DLL_LINKAGE toUnicode(const std::string & text);
 	std::string DLL_LINKAGE toUnicode(const std::string & text, const std::string & encoding);
 
 	/// converts text from unicode to specified encoding or to one specified in settings
 	/// NOTE: usage of these functions should be avoided if possible
-	std::string DLL_LINKAGE fromUnicode(const std::string & text);
 	std::string DLL_LINKAGE fromUnicode(const std::string & text, const std::string & encoding);
 
 	///delete specified amount of UTF-8 characters from right

lib/filesystem/CBinaryReader.cpp

@@ -86,7 +86,7 @@ INSTANTIATE(si64, readInt64)
 
 #undef INSTANTIATE
 
-std::string CBinaryReader::readString()
+std::string CBinaryReader::readBaseString()
 {
 	unsigned int len = readUInt32();
 	assert(len <= 500000); //not too long
@@ -95,10 +95,7 @@ std::string CBinaryReader::readString()
 		std::string ret;
 		ret.resize(len);
 		read(reinterpret_cast<ui8*>(&ret[0]), len);
-		//FIXME: any need to move this into separate "read localized string" method?
-		if (TextOperations::isValidASCII(ret))
-			return ret;
-		return TextOperations::toUnicode(ret);
+		return ret;
 	}
 	return "";
 

lib/filesystem/CBinaryReader.h

@@ -73,7 +73,8 @@ public:
 	ui64 readUInt64();
 	si64 readInt64();
 
-	std::string readString();
+	/// Reads string without any encoding conversions
+	std::string readBaseString();
 
 	inline bool readBool()
 	{

lib/mapObjects/ObjectTemplate.cpp

@@ -198,7 +198,7 @@ void ObjectTemplate::readMsk()
 
 void ObjectTemplate::readMap(CBinaryReader & reader)
 {
-	animationFile = reader.readString();
+	animationFile = reader.readBaseString();
 
 	setSize(8, 6);
 	ui8 blockMask[6];

lib/mapping/CCampaignHandler.cpp

@@ -110,14 +110,19 @@ std::unique_ptr<CCampaign> CCampaignHandler::getCampaign( const std::string & na
 	return ret;
 }
 
+std::string CCampaignHandler::readLocalizedString(CBinaryReader & reader)
+{
+	return reader.readBaseString();
+}
+
 CCampaignHeader CCampaignHandler::readHeaderFromMemory( CBinaryReader & reader )
 {
 	CCampaignHeader ret;
 
 	ret.version = reader.readUInt32();
 	ret.mapVersion = reader.readUInt8() - 1;//change range of it from [1, 20] to [0, 19]
-	ret.name = reader.readString();
-	ret.description = reader.readString();
+	ret.name = readLocalizedString(reader);
+	ret.description = readLocalizedString(reader);
 	if (ret.version > CampaignVersion::RoE)
 		ret.difficultyChoosenByPlayer = reader.readInt8();
 	else
@@ -136,14 +141,14 @@ CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & read
 		{
 			ret.prologVideo = reader.readUInt8();
 			ret.prologMusic = reader.readUInt8();
-			ret.prologText = reader.readString();
+			ret.prologText = readLocalizedString(reader);
 		}
 		return ret;
 	};
 
 	CCampaignScenario ret;
 	ret.conquered = false;
-	ret.mapName = reader.readString();
+	ret.mapName = readLocalizedString(reader);
 	ret.packedMapSize = reader.readUInt32();
 	if(mapVersion == 18)//unholy alliance
 	{
@@ -155,7 +160,7 @@ CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & read
 	}
 	ret.regionColor = reader.readUInt8();
 	ret.difficulty = reader.readUInt8();
-	ret.regionText = reader.readString();
+	ret.regionText = readLocalizedString(reader);
 	ret.prolog = prologEpilogReader();
 	ret.epilog = prologEpilogReader();
 

lib/mapping/CCampaignHandler.h

@@ -218,6 +218,8 @@ class DLL_LINKAGE CCampaignHandler
 {
 	std::vector<size_t> scenariosCountPerCampaign;
 
+	static std::string readLocalizedString(CBinaryReader & reader);
+
 	static CCampaignHeader readHeaderFromMemory(CBinaryReader & reader);
 	static CCampaignScenario readScenarioFromMemory(CBinaryReader & reader, int version, int mapVersion );
 	static CScenarioTravel readScenarioTravelFromMemory(CBinaryReader & reader, int version);

lib/mapping/MapFormatH3M.h

@@ -17,11 +17,11 @@
 
 #include "../int3.h"
 
-#include "../filesystem/CBinaryReader.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 
 class CGHeroInstance;
+class CBinaryReader;
 class CArtifactInstance;
 class CGObjectInstance;
 class CGSeerHut;
@@ -234,14 +234,10 @@ private:
 	/**
 	* Helper to read map position
 	*/
-	inline int3 readInt3()
-	{
-		int3 p;
-		p.x = reader.readUInt8();
-		p.y = reader.readUInt8();
-		p.z = reader.readUInt8();
-		return p;
-	}
+	int3 readInt3();
+
+	/// reads string from input stream and converts it to unicode
+	std::string readLocalizedString();
 
 	void afterRead();
 
@@ -257,8 +253,7 @@ private:
 	 * (when loading a map then the mapHeader ptr points to the same object)
 	 */
 	std::unique_ptr<CMapHeader> mapHeader;
-
-	CBinaryReader reader;
+	std::unique_ptr<CBinaryReader> reader;
 	CInputStream * inputStream;
 
 };