Merge pull request #1563 from IvanSavenko/translation_validation

Translation validation for mods

Global.h

@@ -730,28 +730,7 @@ namespace vstd
 		return a + (b - a) * f;
 	}
 
-	/// converts number into string using metric system prefixes, e.g. 'k' or 'M' to keep resulting strings within specified size
-	/// Note that resulting string may have more symbols than digits: minus sign and prefix symbol
-	template<typename Arithmetic>
-	std::string formatMetric(Arithmetic number, int maxDigits)
-	{
-		Arithmetic max = std::pow(10, maxDigits);
-		if (std::abs(number) < max)
-			return std::to_string(number);
-
-		std::string symbols = " kMGTPE";
-		auto iter = symbols.begin();
-
-		while (std::abs(number) >= max)
-		{
-			number /= 1000;
-			iter++;
 
-			assert(iter != symbols.end());//should be enough even for int64
-		}
-		return std::to_string(number) + *iter;
-	}
-	
 	///compile-time version of std::abs for ints for int3, in clang++15 std::abs is constexpr
 	static constexpr int abs(int i) {
 		if(i < 0) return -i;

Mods/vcmi/config/vcmi/ukrainian.json

@@ -13,6 +13,9 @@
 	"vcmi.adventureMap.monsterThreat.levels.10" : "Смертельна",
 	"vcmi.adventureMap.monsterThreat.levels.11" : "Неможлива",
 
+	"vcmi.adventureMap.moveCostDetails" : "Очки руху - Вартість: %TURNS ходів + %POINTS очок. Залишок очок: %REMAINING",
+	"vcmi.adventureMap.moveCostDetailsNoTurns" : "Очки руху - Вартість: %POINTS очок, Залишок очок: %REMAINING",
+
 	"vcmi.adventureMap.confirmRestartGame"  : "Ви впевнені, що хочете перезапустити гру?",
 	"vcmi.adventureMap.noTownWithMarket"    : "Немає доступних ринків!",
 	"vcmi.adventureMap.noTownWithTavern"    : "Немає доступного міста з таверною!",
@@ -222,4 +225,19 @@
 	"core.bonus.WATER_IMMUNITY.description" : "Імунітет до всіх заклять школи Води",
 	"core.bonus.WIDE_BREATH.name" : "Широкий подих",
 	"core.bonus.WIDE_BREATH.description" : "Атака широким подихом",
+	"core.bonus.LIMITED_SHOOTING_RANGE.name" : "Обмежена дальність стрільби",
+	"core.bonus.LIMITED_SHOOTING_RANGE.description" : "Не може стріляти по цілях на відстані більше ${val} гексів",
+	
+	"vcmi.stackExperience.description" : "» S t a c k   E x p e r i e n c e   D e t a i l s «\n\nCreature Type ................... : %s\nExperience Rank ................. : %s (%i)\nExperience Points ............... : %i\nExperience Points to Next Rank .. : %i\nMaximum Experience per Battle ... : %i%% (%i)\nNumber of Creatures in stack .... : %i\nMaximum New Recruits\n without losing current Rank .... : %i\nExperience Multiplier ........... : %.2f\nUpgrade Multiplier .............. : %.2f\nExperience after Rank 10 ........ : %i\nMaximum New Recruits to remain at\n Rank 10 if at Maximum Experience : %i",
+	"vcmi.stackExperience.rank.1" :  "Початковий",
+	"vcmi.stackExperience.rank.2" :  "Новачок",
+	"vcmi.stackExperience.rank.3" :  "Підготовлений",
+	"vcmi.stackExperience.rank.4" :  "Досвідчений",
+	"vcmi.stackExperience.rank.5" :  "Випробуваний",
+	"vcmi.stackExperience.rank.6" :  "Ветеран",
+	"vcmi.stackExperience.rank.7" :  "Адепт",
+	"vcmi.stackExperience.rank.8" :  "Експерт",
+	"vcmi.stackExperience.rank.9" :  "Еліта",
+	"vcmi.stackExperience.rank.10" : "Майстер",
+	"vcmi.stackExperience.rank.11" : "Профі",
 }

client/adventureMap/CInGameConsole.cpp

@@ -20,7 +20,7 @@
 
 #include "../../CCallback.h"
 #include "../../lib/CConfigHandler.h"
-#include "../../lib/CGeneralTextHandler.h"
+#include "../../lib/TextOperations.h"
 #include "../../lib/mapObjects/CArmedInstance.h"
 
 #include <SDL_timer.h>
@@ -127,7 +127,7 @@ void CInGameConsole::keyPressed (const SDL_Keycode & key)
 		{
 			if(enteredText.size() > 1)
 			{
-				Unicode::trimRight(enteredText,2);
+				TextOperations::trimRightUnicode(enteredText,2);
 				enteredText += '_';
 				refreshEnteredText();
 			}

client/battle/BattleInterfaceClasses.cpp

@@ -47,6 +47,7 @@
 #include "../../lib/StartInfo.h"
 #include "../../lib/CondSh.h"
 #include "../../lib/mapObjects/CGTownInstance.h"
+#include "../../lib/TextOperations.h"
 
 void BattleConsole::showAll(SDL_Surface * to)
 {
@@ -717,7 +718,7 @@ void StackQueue::StackBox::setUnit(const battle::Unit * unit, size_t turn)
 		if (unit->unitType()->idNumber == CreatureID::ARROW_TOWERS)
 			icon->setFrame(owner->getSiegeShooterIconID(), 1);
 
-		amount->setText(vstd::formatMetric(unit->getCount(), 4));
+		amount->setText(TextOperations::formatMetric(unit->getCount(), 4));
 
 		if(stateIcon)
 		{

client/battle/BattleStacksController.cpp

@@ -36,6 +36,7 @@
 #include "../../lib/CGameState.h"
 #include "../../lib/CStack.h"
 #include "../../lib/CondSh.h"
+#include "../../lib/TextOperations.h"
 
 static void onAnimationFinished(const CStack *stack, std::weak_ptr<CreatureAnimation> anim)
 {
@@ -316,7 +317,7 @@ void BattleStacksController::showStackAmountBox(Canvas & canvas, const CStack *
 	//blitting amount
 	Point textPos = stackAnimation[stack->ID]->pos.topLeft() + amountBG->dimensions()/2 + Point(xAdd, yAdd);
 
-	canvas.drawText(textPos, EFonts::FONT_TINY, Colors::WHITE, ETextAlignment::CENTER, vstd::formatMetric(stack->getCount(), 4));
+	canvas.drawText(textPos, EFonts::FONT_TINY, Colors::WHITE, ETextAlignment::CENTER, TextOperations::formatMetric(stack->getCount(), 4));
 }
 
 void BattleStacksController::showStack(Canvas & canvas, const CStack * stack)

client/lobby/CCampaignInfoScreen.cpp

@@ -15,6 +15,7 @@
 #include "../../lib/CGeneralTextHandler.h"
 #include "../../lib/StartInfo.h"
 #include "../../lib/mapping/CMapInfo.h"
+#include "../../lib/mapping/CMap.h"
 #include "../gui/CGuiHandler.h"
 #include "../CGameInfo.h"
 #include "../CPlayerInterface.h"

client/lobby/CSavingScreen.cpp

@@ -24,6 +24,7 @@
 #include "../../lib/StartInfo.h"
 #include "../../lib/filesystem/Filesystem.h"
 #include "../../lib/mapping/CMapInfo.h"
+#include "../../lib/mapping/CMap.h"
 
 CSavingScreen::CSavingScreen()
 	: CSelectionBase(ESelectionScreen::saveGame)

client/lobby/RandomMapTab.cpp

@@ -25,6 +25,7 @@
 
 #include "../../lib/CGeneralTextHandler.h"
 #include "../../lib/mapping/CMapInfo.h"
+#include "../../lib/mapping/CMap.h"
 #include "../../lib/rmg/CMapGenOptions.h"
 #include "../../lib/CModHandler.h"
 #include "../../lib/rmg/CRmgTemplateStorage.h"

client/lobby/SelectionTab.cpp

@@ -33,6 +33,8 @@
 #include "../../lib/CModHandler.h"
 #include "../../lib/filesystem/Filesystem.h"
 #include "../../lib/mapping/CMapInfo.h"
+#include "../../lib/mapping/CMap.h"
+#include "../../lib/mapping/CCampaignHandler.h"
 #include "../../lib/serializer/Connection.h"
 
 bool mapSorter::operator()(const std::shared_ptr<CMapInfo> aaa, const std::shared_ptr<CMapInfo> bbb)

client/render/IFont.cpp

@@ -12,14 +12,14 @@
 #include "IFont.h"
 
 #include "../../lib/Point.h"
-#include "../../lib/CGeneralTextHandler.h"
+#include "../../lib/TextOperations.h"
 //
 
 size_t IFont::getStringWidth(const std::string & data) const
 {
 	size_t width = 0;
 
-	for(size_t i=0; i<data.size(); i += Unicode::getCharacterSize(data[i]))
+	for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
 	{
 		width += getGlyphWidth(data.data() + i);
 	}

client/renderSDL/CBitmapFont.cpp

@@ -11,58 +11,86 @@
 #include "CBitmapFont.h"
 
 #include "SDL_Extensions.h"
+#include "../CGameInfo.h"
+#include "../render/Colors.h"
 
-#include "../../lib/vcmi_endian.h"
-#include "../../lib/filesystem/Filesystem.h"
-#include "../../lib/CGeneralTextHandler.h"
+#include "../../lib/CModHandler.h"
+#include "../../lib/Languages.h"
 #include "../../lib/Rect.h"
+#include "../../lib/TextOperations.h"
+#include "../../lib/filesystem/Filesystem.h"
+#include "../../lib/vcmi_endian.h"
 
 #include <SDL_surface.h>
 
-std::array<CBitmapFont::BitmapChar, CBitmapFont::totalChars> CBitmapFont::loadChars() const
+void CBitmapFont::loadModFont(const std::string & modName, const ResourceID & resource)
 {
-	std::array<BitmapChar, totalChars> ret;
+	auto data = CResourceHandler::get(modName)->load(resource)->readAll();
+	std::string modLanguage = CGI->modh->getModLanguage(modName);
+	std::string modEncoding = Languages::getLanguageOptions(modLanguage).encoding;
 
-	size_t offset = 32;
+	uint32_t dataHeight = data.first[5];
 
-	for (auto & elem : ret)
-	{
-		elem.leftOffset =  read_le_u32(data.first.get() + offset); offset+=4;
-		elem.width =       read_le_u32(data.first.get() + offset); offset+=4;
-		elem.rightOffset = read_le_u32(data.first.get() + offset); offset+=4;
-	}
+	maxHeight = std::max(maxHeight, dataHeight);
 
-	for (auto & elem : ret)
+	constexpr size_t symbolsInFile = 0x100;
+	constexpr size_t baseIndex = 32;
+	constexpr size_t offsetIndex = baseIndex + symbolsInFile*12;
+	constexpr size_t dataIndex = offsetIndex + symbolsInFile*4;
+
+	for (uint32_t charIndex = 0; charIndex < symbolsInFile; ++charIndex)
 	{
-		int pixelOffset =  read_le_u32(data.first.get() + offset); offset+=4;
-		elem.pixels = data.first.get() + 4128 + pixelOffset;
+		CodePoint codepoint = TextOperations::getUnicodeCodepoint(static_cast<char>(charIndex), modEncoding);
+
+		BitmapChar symbol;
+
+		symbol.leftOffset =  read_le_u32(data.first.get() + baseIndex + charIndex * 12 + 0);
+		symbol.width =       read_le_u32(data.first.get() + baseIndex + charIndex * 12 + 4);
+		symbol.rightOffset = read_le_u32(data.first.get() + baseIndex + charIndex * 12 + 8);
+		symbol.height = dataHeight;
+
+		uint32_t pixelDataOffset = read_le_u32(data.first.get() + offsetIndex + charIndex * 4);
+		uint32_t pixelsCount = dataHeight * symbol.width;
+
+		symbol.pixels.resize(pixelsCount);
+
+		uint8_t * pixelData = data.first.get() + dataIndex + pixelDataOffset;
 
-		assert(pixelOffset + 4128 < data.second);
+		std::copy_n(pixelData, pixelsCount, symbol.pixels.data() );
+
+		chars[codepoint] = symbol;
 	}
-	return ret;
 }
 
 CBitmapFont::CBitmapFont(const std::string & filename):
-	data(CResourceHandler::get()->load(ResourceID("data/" + filename, EResType::BMP_FONT))->readAll()),
-	chars(loadChars()),
-	height(data.first.get()[5])
-{}
+	maxHeight(0)
+{
+	ResourceID resource("data/" + filename, EResType::BMP_FONT);
+
+	loadModFont("core", resource);
+
+	for (auto const & modName : VLC->modh->getActiveMods())
+	{
+		if (CResourceHandler::get(modName)->existsResource(resource))
+			loadModFont(modName, resource);
+	}
+}
 
 size_t CBitmapFont::getLineHeight() const
 {
-	return height;
+	return maxHeight;
 }
 
 size_t CBitmapFont::getGlyphWidth(const char * data) const
 {
-	std::string localChar = Unicode::fromUnicode(std::string(data, Unicode::getCharacterSize(data[0])));
+	CodePoint localChar = TextOperations::getUnicodeCodepoint(data, 4);
 
-	if (localChar.size() == 1)
-	{
-		const BitmapChar & ch = chars[ui8(localChar[0])];
-		return ch.leftOffset + ch.width + ch.rightOffset;
-	}
-	return 0;
+	auto iter = chars.find(localChar);
+
+	if (iter == chars.end())
+		return 0;
+
+	return iter->second.leftOffset + iter->second.width + iter->second.rightOffset;
 }
 
 void CBitmapFont::renderCharacter(SDL_Surface * surface, const BitmapChar & character, const SDL_Color & color, int &posX, int &posY) const
@@ -78,7 +106,7 @@ void CBitmapFont::renderCharacter(SDL_Surface * surface, const BitmapChar & char
 
 	// start of line, may differ from 0 due to end of surface or clipped surface
 	int lineBegin = std::max<int>(0, clipRect.y - posY);
-	int lineEnd   = std::min<int>(height, clipRect.y + clipRect.h - posY - 1);
+	int lineEnd   = std::min<int>(character.height, clipRect.y + clipRect.h - posY - 1);
 
 	// start end end of each row, may differ from 0
 	int rowBegin = std::max<int>(0, clipRect.x - posX);
@@ -88,7 +116,7 @@ void CBitmapFont::renderCharacter(SDL_Surface * surface, const BitmapChar & char
 	for(int dy = lineBegin; dy <lineEnd; dy++)
 	{
 		uint8_t *dstLine = (uint8_t*)surface->pixels;
-		uint8_t *srcLine = character.pixels;
+		const uint8_t *srcLine = character.pixels.data();
 
 		// shift source\destination pixels to current position
 		dstLine += (posY+dy) * surface->pitch + posX * bpp;
@@ -131,12 +159,14 @@ void CBitmapFont::renderText(SDL_Surface * surface, const std::string & data, co
 
 	SDL_LockSurface(surface);
 
-	for(size_t i=0; i<data.size(); i += Unicode::getCharacterSize(data[i]))
+	for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
 	{
-		std::string localChar = Unicode::fromUnicode(data.substr(i, Unicode::getCharacterSize(data[i])));
+		CodePoint codepoint = TextOperations::getUnicodeCodepoint(data.data() + i, data.size() - i);
+
+		auto iter = chars.find(codepoint);
 
-		if (localChar.size() == 1)
-			renderCharacter(surface, chars[ui8(localChar[0])], color, posX, posY);
+		if (iter != chars.end())
+			renderCharacter(surface, iter->second, color, posX, posY);
 	}
 	SDL_UnlockSurface(surface);
 }

client/renderSDL/CBitmapFont.h

@@ -11,30 +11,32 @@
 
 #include "../render/IFont.h"
 
+VCMI_LIB_NAMESPACE_BEGIN
+class ResourceID;
+VCMI_LIB_NAMESPACE_END
+
 class CBitmapFont : public IFont
 {
-	static const size_t totalChars = 256;
+	using CodePoint = uint32_t;
 
 	struct BitmapChar
 	{
-		si32 leftOffset;
-		ui32 width;
-		si32 rightOffset;
-		ui8 *pixels; // pixels of this character, part of BitmapFont::data
+		int32_t leftOffset;
+		uint32_t width;
+		uint32_t height;
+		int32_t rightOffset;
+		std::vector<uint8_t> pixels; // pixels of this character, part of BitmapFont::data
 	};
 
-	const std::pair<std::unique_ptr<ui8[]>, ui64> data;
-
-	const std::array<BitmapChar, totalChars> chars;
-	const ui8 height;
+	std::unordered_map<CodePoint, BitmapChar> chars;
+	uint32_t maxHeight;
 
-	std::array<BitmapChar, totalChars> loadChars() const;
+	void loadModFont(const std::string & modName, const ResourceID & resource);
 
 	void renderCharacter(SDL_Surface * surface, const BitmapChar & character, const SDL_Color & color, int &posX, int &posY) const;
-
 	void renderText(SDL_Surface * surface, const std::string & data, const SDL_Color & color, const Point & pos) const override;
 public:
-	CBitmapFont(const std::string & filename);
+	explicit CBitmapFont(const std::string & filename);
 
 	size_t getLineHeight() const override;
 	size_t getGlyphWidth(const char * data) const override;

client/renderSDL/CBitmapHanFont.cpp

@@ -15,7 +15,7 @@
 
 #include "../../lib/JsonNode.h"
 #include "../../lib/filesystem/Filesystem.h"
-#include "../../lib/CGeneralTextHandler.h"
+#include "../../lib/TextOperations.h"
 #include "../../lib/Rect.h"
 
 #include <SDL_surface.h>
@@ -83,9 +83,9 @@ void CBitmapHanFont::renderText(SDL_Surface * surface, const std::string & data,
 
 	SDL_LockSurface(surface);
 
-	for(size_t i=0; i<data.size(); i += Unicode::getCharacterSize(data[i]))
+	for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
 	{
-		std::string localChar = Unicode::fromUnicode(data.substr(i, Unicode::getCharacterSize(data[i])));
+		std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])), "GBK");
 
 		if (localChar.size() == 1)
 			fallback->renderCharacter(surface, fallback->chars[ui8(localChar[0])], color, posX, posY);
@@ -115,7 +115,7 @@ size_t CBitmapHanFont::getLineHeight() const
 
 size_t CBitmapHanFont::getGlyphWidth(const char * data) const
 {
-	std::string localChar = Unicode::fromUnicode(std::string(data, Unicode::getCharacterSize(data[0])));
+	std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])), "GBK");
 
 	if (localChar.size() == 1)
 		return fallback->getGlyphWidth(data);

client/renderSDL/CTrueTypeFont.cpp

@@ -14,7 +14,7 @@
 #include "../renderSDL/SDL_Extensions.h"
 
 #include "../../lib/JsonNode.h"
-#include "../../lib/CGeneralTextHandler.h"
+#include "../../lib/TextOperations.h"
 #include "../../lib/filesystem/Filesystem.h"
 
 #include <SDL_ttf.h>
@@ -66,7 +66,7 @@ size_t CTrueTypeFont::getLineHeight() const
 
 size_t CTrueTypeFont::getGlyphWidth(const char *data) const
 {
-	return getStringWidth(std::string(data, Unicode::getCharacterSize(*data)));
+	return getStringWidth(std::string(data, TextOperations::getUnicodeCharacterSize(*data)));
 	/*
 	int advance;
 	TTF_GlyphMetrics(font.get(), *data, nullptr, nullptr, nullptr, nullptr, &advance);

client/widgets/CGarrisonInt.cpp

@@ -25,6 +25,7 @@
 #include "../../lib/CGeneralTextHandler.h"
 #include "../../lib/CCreatureHandler.h"
 #include "../../lib/mapObjects/CGHeroInstance.h"
+#include "../../lib/TextOperations.h"
 
 #include "../../lib/CGameState.h"
 
@@ -374,7 +375,7 @@ void CGarrisonSlot::update()
 		creatureImage->setFrame(creature->getIconIndex());
 
 		stackCount->enable();
-		stackCount->setText(vstd::formatMetric(myStack->count, 4));
+		stackCount->setText(TextOperations::formatMetric(myStack->count, 4));
 	}
 	else
 	{

client/widgets/MiscWidgets.cpp

@@ -24,12 +24,13 @@
 
 #include "../../CCallback.h"
 
-#include "../../lib/mapObjects/CGHeroInstance.h"
-#include "../../lib/mapObjects/CGTownInstance.h"
+#include "../../lib/CConfigHandler.h"
+#include "../../lib/CGameState.h"
 #include "../../lib/CGeneralTextHandler.h"
 #include "../../lib/CModHandler.h"
-#include "../../lib/CGameState.h"
-#include "../../lib/CConfigHandler.h"
+#include "../../lib/TextOperations.h"
+#include "../../lib/mapObjects/CGHeroInstance.h"
+#include "../../lib/mapObjects/CGTownInstance.h"
 
 void CHoverableArea::hover (bool on)
 {
@@ -251,7 +252,7 @@ void CArmyTooltip::init(const InfoAboutArmy &army)
 		std::string subtitle;
 		if(army.army.isDetailed)
 		{
-			subtitle = vstd::formatMetric(slot.second.count, 4);
+			subtitle = TextOperations::formatMetric(slot.second.count, 4);
 		}
 		else
 		{

client/widgets/TextControls.cpp

@@ -19,7 +19,7 @@
 #include "../adventureMap/CInGameConsole.h"
 #include "../renderSDL/SDL_Extensions.h"
 
-#include "../../lib/CGeneralTextHandler.h"
+#include "../../lib/TextOperations.h"
 
 #ifdef VCMI_ANDROID
 #include "lib/CAndroidVMHelper.h"
@@ -570,12 +570,12 @@ void CTextInput::keyPressed(const SDL_Keycode & key)
 	case SDLK_BACKSPACE:
 		if(!newText.empty())
 		{
-			Unicode::trimRight(newText);
+			TextOperations::trimRightUnicode(newText);
 			redrawNeeded = true;
 		}
 		else if(!text.empty())
 		{
-			Unicode::trimRight(text);
+			TextOperations::trimRightUnicode(text);
 			redrawNeeded = true;
 		}
 		break;

client/windows/CCreatureWindow.cpp

@@ -31,6 +31,7 @@
 #include "../../lib/CModHandler.h"
 #include "../../lib/CHeroHandler.h"
 #include "../../lib/CGameState.h"
+#include "../../lib/TextOperations.h"
 
 class CCreatureArtifactInstance;
 class CSelectableSkill;
@@ -582,7 +583,7 @@ CStackWindow::MainSection::MainSection(CStackWindow * owner, int yOffset, bool s
 		}
 		expLabel = std::make_shared<CLabel>(
 				pos.x + 21, pos.y + 52, FONT_SMALL, ETextAlignment::CENTER, Colors::WHITE,
-				vstd::formatMetric(stack->experience, 6));
+				TextOperations::formatMetric(stack->experience, 6));
 	}
 
 	if(showArt)

client/windows/CMessage.cpp

@@ -13,6 +13,7 @@
 
 #include "../CGameInfo.h"
 #include "../../lib/CGeneralTextHandler.h"
+#include "../../lib/TextOperations.h"
 
 #include "../windows/InfoWindows.h"
 #include "../widgets/Buttons.h"
@@ -132,7 +133,7 @@ std::vector<std::string> CMessage::breakText( std::string text, size_t maxLineWi
 		// loops till line is full or end of text reached
 		while(currPos < text.length()  &&  text[currPos] != 0x0a  &&  lineWidth < maxLineWidth)
 		{
-			symbolSize = Unicode::getCharacterSize(text[currPos]);
+			symbolSize = TextOperations::getUnicodeCharacterSize(text[currPos]);
 			glyphWidth = graphics->fonts[font]->getGlyphWidth(text.data() + currPos);
 
 			// candidate for line break

client/windows/GUIClasses.cpp

@@ -63,6 +63,7 @@
 #include "../lib/mapping/CMap.h"
 #include "../lib/NetPacksBase.h"
 #include "../lib/StartInfo.h"
+#include "../lib/TextOperations.h"
 
 #include <SDL_surface.h>
 
@@ -407,8 +408,12 @@ CLevelWindow::CLevelWindow(const CGHeroInstance * hero, PrimarySkill::PrimarySki
 	mainTitle = std::make_shared<CLabel>(192, 33, FONT_MEDIUM, ETextAlignment::CENTER, Colors::WHITE, boost::str(boost::format(CGI->generaltexth->allTexts[444]) % hero->getNameTranslated()));
 
 	//%s is now a level %d %s.
-	levelTitle = std::make_shared<CLabel>(192, 162, FONT_MEDIUM, ETextAlignment::CENTER, Colors::WHITE,
-		boost::str(boost::format(CGI->generaltexth->allTexts[445]) % hero->getNameTranslated() % hero->level % hero->type->heroClass->getNameTranslated()));
+	std::string levelTitleText = CGI->generaltexth->translate("core.genrltxt.445");
+	boost::replace_first(levelTitleText, "%s", hero->getNameTranslated());
+	boost::replace_first(levelTitleText, "%d", std::to_string(hero->level));
+	boost::replace_first(levelTitleText, "%s", hero->type->heroClass->getNameTranslated());
+
+	levelTitle = std::make_shared<CLabel>(192, 162, FONT_MEDIUM, ETextAlignment::CENTER, Colors::WHITE, levelTitleText);
 
 	skillIcon = std::make_shared<CAnimImage>("PSKIL42", pskill, 0, 174, 190);
 
@@ -1066,8 +1071,8 @@ void CExchangeWindow::updateWidgets()
 			secSkillIcons[leftRight][m]->setFrame(2 + id * 3 + level);
 		}
 
-		expValues[leftRight]->setText(vstd::formatMetric(hero->exp, 3));
-		manaValues[leftRight]->setText(vstd::formatMetric(hero->mana, 3));
+		expValues[leftRight]->setText(TextOperations::formatMetric(hero->exp, 3));
+		manaValues[leftRight]->setText(TextOperations::formatMetric(hero->mana, 3));
 
 		morale[leftRight]->set(hero);
 		luck[leftRight]->set(hero);

cmake_modules/VCMI_lib.cmake

@@ -197,6 +197,7 @@ macro(add_main_lib TARGET_NAME LIBRARY_TYPE)
 		${MAIN_LIB_DIR}/RoadHandler.cpp
 		${MAIN_LIB_DIR}/ScriptHandler.cpp
 		${MAIN_LIB_DIR}/TerrainHandler.cpp
+		${MAIN_LIB_DIR}/TextOperations.cpp
 		${MAIN_LIB_DIR}/VCMIDirs.cpp
 		${MAIN_LIB_DIR}/VCMI_Lib.cpp
 	)
@@ -434,6 +435,7 @@ macro(add_main_lib TARGET_NAME LIBRARY_TYPE)
 		${MAIN_LIB_DIR}/Interprocess.h
 		${MAIN_LIB_DIR}/JsonDetail.h
 		${MAIN_LIB_DIR}/JsonNode.h
+		${MAIN_LIB_DIR}/Languages.h
 		${MAIN_LIB_DIR}/LoadProgress.h
 		${MAIN_LIB_DIR}/LogicalExpression.h
 		${MAIN_LIB_DIR}/NetPacksBase.h
@@ -453,6 +455,7 @@ macro(add_main_lib TARGET_NAME LIBRARY_TYPE)
 		${MAIN_LIB_DIR}/StartInfo.h
 		${MAIN_LIB_DIR}/StringConstants.h
 		${MAIN_LIB_DIR}/TerrainHandler.h
+		${MAIN_LIB_DIR}/TextOperations.h
 		${MAIN_LIB_DIR}/UnlockGuard.h
 		${MAIN_LIB_DIR}/VCMIDirs.h
 		${MAIN_LIB_DIR}/vcmi_endian.h

config/objects/generic.json

@@ -840,6 +840,10 @@
 			}
 		}
 	},
+	"marketOfTime" : { // Unused/not implemented H3 object present on some maps RoE maps
+		"index" :50,
+		"handler": "generic"
+	},
 	"tavern" : {
 		"index" :95,
 		"handler": "generic",

config/schemas/mod.json

@@ -8,7 +8,7 @@
 		"localizable" : {
 			"type":"object",
 			"additionalProperties" : false,
-			"required" : [ "name", "description", "modType" ],
+			"required" : [ "name", "description" ],
 			"properties":{
 				"name": {
 					"type":"string",
@@ -84,6 +84,11 @@
 			"description": "Home page of mod or link to forum thread"
 		},
 
+		"language" : {
+			"type":"string",
+			"description": "Base language of the mod, before applying localizations. By default vcmi assumes English",
+			"enum" : [ "english", "german", "polish", "russian", "ukrainian" ],
+		},
 		"depends": {
 			"type":"array",
 			"description": "List of mods that are required to run this one",

lib/CArtHandler.cpp

@@ -346,9 +346,9 @@ CArtifact * CArtHandler::loadFromJson(const std::string & scope, const JsonNode
 
 	const JsonNode & text = node["text"];
 
-	VLC->generaltexth->registerString(art->getNameTextID(), text["name"].String());
-	VLC->generaltexth->registerString(art->getDescriptionTextID(), text["description"].String());
-	VLC->generaltexth->registerString(art->getEventTextID(), text["event"].String());
+	VLC->generaltexth->registerString(scope, art->getNameTextID(), text["name"].String());
+	VLC->generaltexth->registerString(scope, art->getDescriptionTextID(), text["description"].String());
+	VLC->generaltexth->registerString(scope, art->getEventTextID(), text["event"].String());
 
 	const JsonNode & graphics = node["graphics"];
 	art->image = graphics["image"].String();

lib/CBonusTypeHandler.cpp

@@ -240,8 +240,8 @@ void CBonusTypeHandler::loadItem(const JsonNode & source, CBonusType & dest, con
 
 	if (!dest.hidden)
 	{
-		VLC->generaltexth->registerString(dest.getNameTextID(), source["name"].String());
-		VLC->generaltexth->registerString(dest.getDescriptionTextID(), source["description"].String());
+		VLC->generaltexth->registerString( "core", dest.getNameTextID(), source["name"].String());
+		VLC->generaltexth->registerString( "core", dest.getDescriptionTextID(), source["description"].String());
 	}
 
 	const JsonNode & graphics = source["graphics"];

lib/CCreatureHandler.cpp

@@ -636,8 +636,8 @@ CCreature * CCreatureHandler::loadFromJson(const std::string & scope, const Json
 
 	cre->cost = Res::ResourceSet(node["cost"]);
 
-	VLC->generaltexth->registerString(cre->getNameSingularTextID(), node["name"]["singular"].String());
-	VLC->generaltexth->registerString(cre->getNamePluralTextID(), node["name"]["plural"].String());
+	VLC->generaltexth->registerString(scope, cre->getNameSingularTextID(), node["name"]["singular"].String());
+	VLC->generaltexth->registerString(scope, cre->getNamePluralTextID(), node["name"]["plural"].String());
 
 	cre->addBonus(node["hitPoints"].Integer(), Bonus::STACK_HEALTH);
 	cre->addBonus(node["speed"].Integer(), Bonus::STACKS_SPEED);

lib/CGeneralTextHandler.cpp

@@ -10,130 +10,48 @@
 #include "StdInc.h"
 #include "CGeneralTextHandler.h"
 
-#include <boost/locale.hpp>
-
 #include "filesystem/Filesystem.h"
 #include "CConfigHandler.h"
 #include "CModHandler.h"
-#include "GameConstants.h"
 #include "mapObjects/CQuest.h"
 #include "VCMI_Lib.h"
+#include "Languages.h"
+#include "TextOperations.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 
-size_t Unicode::getCharacterSize(char firstByte)
-{
-	// length of utf-8 character can be determined from 1st byte by counting number of highest bits set to 1:
-	// 0xxxxxxx -> 1 -  ASCII chars
-	// 110xxxxx -> 2
-	// 11110xxx -> 4 - last allowed in current standard
-	// 1111110x -> 6 - last allowed in original standard
-
-	if ((ui8)firstByte < 0x80)
-		return 1; // ASCII
-
-	size_t ret = 0;
-
-	for (size_t i=0; i<8; i++)
-	{
-		if (((ui8)firstByte & (0x80 >> i)) != 0)
-			ret++;
-		else
-			break;
-	}
-	return ret;
-}
-
-bool Unicode::isValidCharacter(const char * character, size_t maxSize)
-{
-	// can't be first byte in UTF8
-	if ((ui8)character[0] >= 0x80 && (ui8)character[0] < 0xC0)
-		return false;
-	// first character must follow rules checked in getCharacterSize
-	size_t size = getCharacterSize((ui8)character[0]);
-
-	if ((ui8)character[0] > 0xF4)
-		return false; // above maximum allowed in standard (UTF codepoints are capped at 0x0010FFFF)
-
-	if (size > maxSize)
-		return false;
-
-	// remaining characters must have highest bit set to 1
-	for (size_t i = 1; i < size; i++)
-	{
-		if (((ui8)character[i] & 0x80) == 0)
-			return false;
-	}
-	return true;
-}
-
-bool Unicode::isValidASCII(const std::string & text)
-{
-	for (const char & ch : text)
-		if (ui8(ch) >= 0x80 )
-			return false;
-	return true;
-}
-
-bool Unicode::isValidASCII(const char * data, size_t size)
-{
-	for (size_t i=0; i<size; i++)
-		if (ui8(data[i]) >= 0x80 )
-			return false;
-	return true;
-}
-
-bool Unicode::isValidString(const std::string & text)
-{
-	for (size_t i=0; i<text.size(); i += getCharacterSize(text[i]))
-	{
-		if (!isValidCharacter(text.data() + i, text.size() - i))
-			return false;
-	}
-	return true;
-}
-
-bool Unicode::isValidString(const char * data, size_t size)
-{
-	for (size_t i=0; i<size; i += getCharacterSize(data[i]))
-	{
-		if (!isValidCharacter(data + i, size - i))
-			return false;
-	}
-	return true;
-}
-
 /// Detects language and encoding of H3 text files based on matching against pregenerated footprints of H3 file
-void CGeneralTextHandler::detectInstallParameters() const
-{
-	struct LanguageFootprint
-	{
-		std::string language;
-		std::string encoding;
-		std::array<double, 16> footprint;
-	};
-
-	static const std::vector<LanguageFootprint> knownFootprints =
-	{
-		{ "English",   "CP1252", { { 0.0559, 0.0000, 0.1983, 0.0051, 0.0222, 0.0183, 0.4596, 0.2405, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000 } } },
-		{ "French",    "CP1252", { { 0.0493, 0.0000, 0.1926, 0.0047, 0.0230, 0.0121, 0.4133, 0.2780, 0.0002, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0259, 0.0008 } } },
-		{ "German",    "CP1252", { { 0.0534, 0.0000, 0.1705, 0.0047, 0.0418, 0.0208, 0.4775, 0.2191, 0.0001, 0.0000, 0.0000, 0.0000, 0.0000, 0.0005, 0.0036, 0.0080 } } },
-		{ "Polish",    "CP1250", { { 0.0534, 0.0000, 0.1701, 0.0067, 0.0157, 0.0133, 0.4328, 0.2540, 0.0001, 0.0043, 0.0000, 0.0244, 0.0000, 0.0000, 0.0181, 0.0071 } } },
-		{ "Russian",   "CP1251", { { 0.0548, 0.0000, 0.1744, 0.0061, 0.0031, 0.0009, 0.0046, 0.0136, 0.0000, 0.0004, 0.0000, 0.0000, 0.0227, 0.0061, 0.4882, 0.2252 } } },
-		{ "Ukrainian", "CP1251", { { 0.0559, 0.0000, 0.1807, 0.0059, 0.0036, 0.0013, 0.0046, 0.0134, 0.0000, 0.0004, 0.0000, 0.0487, 0.0209, 0.0060, 0.4615, 0.1972 } } },
-	};
+void CGeneralTextHandler::detectInstallParameters()
+{
+	using LanguageFootprint = std::array<double, 16>;
+
+	static const std::array<LanguageFootprint, 6> knownFootprints =
+	{ {
+		{ { 0.0559, 0.0000, 0.1983, 0.0051, 0.0222, 0.0183, 0.4596, 0.2405, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000 } },
+		{ { 0.0493, 0.0000, 0.1926, 0.0047, 0.0230, 0.0121, 0.4133, 0.2780, 0.0002, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0259, 0.0008 } },
+		{ { 0.0534, 0.0000, 0.1705, 0.0047, 0.0418, 0.0208, 0.4775, 0.2191, 0.0001, 0.0000, 0.0000, 0.0000, 0.0000, 0.0005, 0.0036, 0.0080 } },
+		{ { 0.0534, 0.0000, 0.1701, 0.0067, 0.0157, 0.0133, 0.4328, 0.2540, 0.0001, 0.0043, 0.0000, 0.0244, 0.0000, 0.0000, 0.0181, 0.0071 } },
+		{ { 0.0548, 0.0000, 0.1744, 0.0061, 0.0031, 0.0009, 0.0046, 0.0136, 0.0000, 0.0004, 0.0000, 0.0000, 0.0227, 0.0061, 0.4882, 0.2252 } },
+		{ { 0.0559, 0.0000, 0.1807, 0.0059, 0.0036, 0.0013, 0.0046, 0.0134, 0.0000, 0.0004, 0.0000, 0.0487, 0.0209, 0.0060, 0.4615, 0.1972 } },
+	} };
+
+	static const std::array<std::string, 6> knownLanguages =
+	{ {
+		"english",
+		"french",
+		"german",
+		"polish",
+		"russian",
+		"ukrainian"
+	} };
 
 	// load file that will be used for footprint generation
 	// this is one of the most text-heavy files in game and consists solely from translated texts
 	auto resource = CResourceHandler::get()->load(ResourceID("DATA/GENRLTXT.TXT", EResType::TEXT));
 
-	std::array<size_t, 256> charCount;
-	std::array<double, 16> footprint;
-	std::vector<double> deviations;
-
-	boost::range::fill(charCount, 0);
-	boost::range::fill(footprint, 0.0);
-	deviations.resize(knownFootprints.size(), 0.0);
+	std::array<size_t, 256> charCount{};
+	std::array<double, 16> footprint{};
+	std::array<double, 6> deviations{};
 
 	auto data = resource->readAll();
 
@@ -146,7 +64,7 @@ void CGeneralTextHandler::detectInstallParameters() const
 	// While this will reduce precision, it should not affect output
 	// since we expect only tiny differences compared to reference footprints
 	for (size_t i = 0; i < 256; ++i)
-		footprint[i/16] += double(charCount[i]) / data.second;
+		footprint[i/16] += static_cast<double>(charCount[i]) / data.second;
 
 	logGlobal->debug("Language footprint: %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f",
 			footprint[0], footprint[1], footprint[2],  footprint[3],  footprint[4],  footprint[5],  footprint[6],  footprint[7],
@@ -156,67 +74,21 @@ void CGeneralTextHandler::detectInstallParameters() const
 	for (size_t i = 0; i < deviations.size(); ++i)
 	{
 		for (size_t j = 0; j < footprint.size(); ++j)
-			deviations[i] += std::abs((footprint[j] - knownFootprints[i].footprint[j]));
+			deviations[i] += std::abs((footprint[j] - knownFootprints[i][j]));
 	}
 
 	size_t bestIndex = boost::range::min_element(deviations) - deviations.begin();
 
 	for (size_t i = 0; i < deviations.size(); ++i)
-		logGlobal->debug("Comparing to %s: %f", knownFootprints[i].language, deviations[i]);
-
-	Settings encoding = settings.write["session"]["encoding"];
-	encoding->String() = knownFootprints[bestIndex].encoding;
-}
-
-std::string Unicode::toUnicode(const std::string &text)
-{
-	return toUnicode(text, CGeneralTextHandler::getInstalledEncoding());
-}
-
-std::string Unicode::toUnicode(const std::string &text, const std::string &encoding)
-{
-	return boost::locale::conv::to_utf<char>(text, encoding);
-}
-
-std::string Unicode::fromUnicode(const std::string & text)
-{
-	return fromUnicode(text, CGeneralTextHandler::getInstalledEncoding());
-}
-
-std::string Unicode::fromUnicode(const std::string &text, const std::string &encoding)
-{
-	return boost::locale::conv::from_utf<char>(text, encoding);
-}
-
-void Unicode::trimRight(std::string & text, const size_t amount)
-{
-	if(text.empty())
-		return;
-	//todo: more efficient algorithm
-	for(int i = 0; i< amount; i++){
-		auto b = text.begin();
-		auto e = text.end();
-		size_t lastLen = 0;
-		size_t len = 0;
-		while (b != e) {
-			lastLen = len;
-			size_t n = getCharacterSize(*b);
-
-			if(!isValidCharacter(&(*b),e-b))
-			{
-				logGlobal->error("Invalid UTF8 sequence");
-				break;//invalid sequence will be trimmed
-			}
+		logGlobal->debug("Comparing to %s: %f", knownLanguages[i], deviations[i]);
 
-			len += n;
-			b += n;
-		}
+	Settings language = settings.write["session"]["language"];
+	language->String() = knownLanguages[bestIndex];
 
-		text.resize(lastLen);
-	}
+	Settings encoding = settings.write["session"]["encoding"];
+	encoding->String() =  Languages::getLanguageOptions(knownLanguages[bestIndex]).encoding;
 }
 
-
 //Helper for string -> float conversion
 class LocaleWithComma: public std::numpunct<char>
 {
@@ -227,20 +99,15 @@ protected:
 	}
 };
 
-CLegacyConfigParser::CLegacyConfigParser(std::string URI)
+CLegacyConfigParser::CLegacyConfigParser(std::string URI):
+	CLegacyConfigParser(CResourceHandler::get()->load(ResourceID(URI, EResType::TEXT)))
 {
-	init(CResourceHandler::get()->load(ResourceID(URI, EResType::TEXT)));
 }
 
 CLegacyConfigParser::CLegacyConfigParser(const std::unique_ptr<CInputStream> & input)
-{
-	init(input);
-}
-
-void CLegacyConfigParser::init(const std::unique_ptr<CInputStream> & input)
 {
 	data.reset(new char[input->getSize()]);
-	input->read((ui8*)data.get(), input->getSize());
+	input->read(reinterpret_cast<uint8_t*>(data.get()), input->getSize());
 
 	curr = data.get();
 	end = curr + input->getSize();
@@ -321,9 +188,9 @@ std::string CLegacyConfigParser::readString()
 {
 	// do not convert strings that are already in ASCII - this will only slow down loading process
 	std::string str = readRawString();
-	if (Unicode::isValidASCII(str))
+	if (TextOperations::isValidASCII(str))
 		return str;
-	return Unicode::toUnicode(str);
+	return TextOperations::toUnicode(str, fileEncoding);
 }
 
 float CLegacyConfigParser::readNumber()
@@ -366,7 +233,7 @@ void CGeneralTextHandler::readToVector(std::string const & sourceID, std::string
 	size_t index = 0;
 	do
 	{
-		registerString({sourceID, index}, parser.readString());
+		registerString( "core", {sourceID, index}, parser.readString());
 		index += 1;
 	}
 	while (parser.endLine());
@@ -374,31 +241,107 @@ void CGeneralTextHandler::readToVector(std::string const & sourceID, std::string
 
 const std::string & CGeneralTextHandler::deserialize(const TextIdentifier & identifier) const
 {
-	if(stringsOverrides.count(identifier.get()))
-		return stringsOverrides.at(identifier.get());
+	if(stringsLocalizations.count(identifier.get()) == 0)
+	{
+		logGlobal->error("Unable to find localization for string '%s'", identifier.get());
+		return identifier.get();
+	}
 
-	if(stringsLocalizations.count(identifier.get()))
-		return stringsLocalizations.at(identifier.get());
+	auto const & entry = stringsLocalizations.at(identifier.get());
 
-	logGlobal->error("Unable to find localization for string '%s'", identifier.get());
-	return identifier.get();
+	if (!entry.overrideValue.empty())
+		return entry.overrideValue;
+	return entry.baseValue;
 }
 
-void CGeneralTextHandler::registerString(const TextIdentifier & UID, const std::string & localized)
+void CGeneralTextHandler::registerString(const std::string & modContext, const TextIdentifier & UID, const std::string & localized)
 {
-	assert(UID.get().find("..") == std::string::npos);
-	stringsLocalizations[UID.get()] = localized;
+	assert(UID.get().find("..") == std::string::npos); // invalid identifier - there is section that was evaluated to empty string
+	//assert(stringsLocalizations.count(UID.get()) == 0); // registering already registered string?
+
+	if (stringsLocalizations.count(UID.get()) > 0)
+	{
+		std::string oldValue = stringsLocalizations[UID.get()].baseValue;
+
+		if (oldValue != localized)
+			logMod->warn("Duplicate registered string '%s' found! Old value: '%s', new value: '%s'", UID.get(), oldValue, localized);
+		return;
+	}
+
+	assert(!modContext.empty());
+	assert(!getModLanguage(modContext).empty());
+
+	StringState result;
+	result.baseLanguage = getModLanguage(modContext);
+	result.baseValue = localized;
+	result.modContext = modContext;
+
+	stringsLocalizations[UID.get()] = result;
 }
 
-void CGeneralTextHandler::registerStringOverride(const TextIdentifier & UID, const std::string & localized)
+void CGeneralTextHandler::registerStringOverride(const std::string & modContext, const std::string & language, const TextIdentifier & UID, const std::string & localized)
 {
-	stringsOverrides[UID.get()] = localized;
+	assert(!modContext.empty());
+	assert(!language.empty());
+
+	// NOTE: implicitly creates entry, intended - strings added by vcmi (and potential UI mods) are not registered anywhere at the moment
+	auto & entry = stringsLocalizations[UID.get()];
+
+	entry.overrideLanguage = language;
+	entry.overrideValue = localized;
+	if (entry.modContext.empty())
+		entry.modContext = modContext;
 }
 
-void CGeneralTextHandler::loadTranslationOverrides(const JsonNode & config)
+bool CGeneralTextHandler::validateTranslation(const std::string & language, const std::string & modContext, const JsonNode & config) const
+{
+	bool allPresent = true;
+
+	for (auto const & string : stringsLocalizations)
+	{
+		if (string.second.modContext != modContext)
+			continue;
+
+		if (string.second.baseLanguage == language && !string.second.baseValue.empty())
+			continue;
+
+		if (config.Struct().count(string.first) > 0)
+			continue;
+
+		if (allPresent)
+			logMod->warn("Translation into language '%s' in mod '%s' is incomplete! Missing lines:", language, modContext);
+
+		std::string currentText;
+		if (string.second.overrideValue.empty())
+			currentText = string.second.baseValue;
+		else
+			currentText = string.second.overrideValue;
+
+		logMod->warn(R"(    "%s" : "%s",)", string.first, TextOperations::escapeString(currentText));
+		allPresent = false;
+	}
+
+	bool allFound = true;
+
+	for (auto const & string : config.Struct())
+	{
+		if (stringsLocalizations.count(string.first) > 0)
+			continue;
+
+		if (allFound)
+			logMod->warn("Translation into language '%s' in mod '%s' has unused lines:", language, modContext);
+
+		logMod->warn(R"(    "%s" : "%s",)", string.first, TextOperations::escapeString(string.second.String()));
+		allFound = false;
+	}
+
+	return allPresent && allFound;
+}
+
+void CGeneralTextHandler::loadTranslationOverrides(const std::string & language, const std::string & modContext, const JsonNode & config)
 {
 	for ( auto const & node : config.Struct())
-		registerStringOverride(node.first, node.second.String());
+		registerStringOverride(modContext, language, node.first, node.second.String());
 }
 
 CGeneralTextHandler::CGeneralTextHandler():
@@ -466,7 +409,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 			std::string line = parser.readString();
 			if(!line.empty())
 			{
-				registerString({"core.randtvrn", index}, line);
+				registerString("core", {"core.randtvrn", index}, line);
 				index += 1;
 			}
 		}
@@ -478,7 +421,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 		size_t index = 0;
 		do
 		{
-			registerString({"core.genrltxt", index}, parser.readString());
+			registerString("core", {"core.genrltxt", index}, parser.readString());
 			index += 1;
 		}
 		while (parser.endLine());
@@ -490,8 +433,8 @@ CGeneralTextHandler::CGeneralTextHandler():
 		{
 			std::string first = parser.readString();
 			std::string second = parser.readString();
-			registerString("core.help." + std::to_string(index) + ".hover", first);
-			registerString("core.help." + std::to_string(index) + ".help",  second);
+			registerString("core", "core.help." + std::to_string(index) + ".hover", first);
+			registerString("core", "core.help." + std::to_string(index) + ".help",  second);
 			index += 1;
 		}
 		while (parser.endLine());
@@ -503,9 +446,9 @@ CGeneralTextHandler::CGeneralTextHandler():
 		{
 			std::string color = parser.readString();
 
-			registerString({"core.plcolors", index}, color);
+			registerString("core", {"core.plcolors", index}, color);
 			color[0] = toupper(color[0]);
-			registerString({"vcmi.capitalColors", index}, color);
+			registerString("core", {"vcmi.capitalColors", index}, color);
 			index += 1;
 		}
 		while (parser.endLine());
@@ -518,13 +461,13 @@ CGeneralTextHandler::CGeneralTextHandler():
 
 		for (size_t i = 0; i < 6; ++i)
 		{
-			registerString({"core.seerhut.empty", i}, parser.readString());
+			registerString("core", {"core.seerhut.empty", i}, parser.readString());
 		}
 		parser.endLine();
 
 		for (size_t i = 0; i < 9; ++i) //9 types of quests
 		{
-			std::string questName = CQuest::missionName(CQuest::Emission(1+i));
+			std::string questName = CQuest::missionName(static_cast<CQuest::Emission>(1+i));
 
 			for (size_t j = 0; j < 5; ++j)
 			{
@@ -533,7 +476,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 				parser.readString(); //front description
 				for (size_t k = 0; k < 6; ++k)
 				{
-					registerString({"core.seerhut.quest", questName, questState, k}, parser.readString());
+					registerString("core", {"core.seerhut.quest", questName, questState, k}, parser.readString());
 				}
 				parser.endLine();
 			}
@@ -541,7 +484,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 
 		for (size_t k = 0; k < 6; ++k) //Time limit
 		{
-			registerString({"core.seerhut.time", k}, parser.readString());
+			registerString("core", {"core.seerhut.time", k}, parser.readString());
 		}
 		parser.endLine();
 
@@ -550,7 +493,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 
 		for (size_t i = 0; i < 48; ++i)
 		{
-			registerString({"core.seerhut.names", i}, parser.readString());
+			registerString("core", {"core.seerhut.names", i}, parser.readString());
 			parser.endLine();
 		}
 	}
@@ -567,7 +510,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 			text = parser.readString();
 			if (!text.empty())
 			{
-				registerString({"core.camptext.names", campaignsCount}, text);
+				registerString("core", {"core.camptext.names", campaignsCount}, text);
 				campaignsCount += 1;
 			}
 		}
@@ -588,7 +531,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 				text = parser.readString();
 				if (!text.empty())
 				{
-					registerString({"core.camptext.regions", std::to_string(campaign), region}, text);
+					registerString("core", {"core.camptext.regions", std::to_string(campaign), region}, text);
 					region += 1;
 				}
 			}
@@ -604,38 +547,28 @@ CGeneralTextHandler::CGeneralTextHandler():
 	}
 }
 
-int32_t CGeneralTextHandler::pluralText(const int32_t textIndex, const int32_t count) const
+int32_t CGeneralTextHandler::pluralText(int32_t textIndex, int32_t count) const
 {
 	if(textIndex == 0)
 		return 0;
-	else if(textIndex < 0)
+	if(textIndex < 0)
 		return -textIndex;
-	else if(count == 1)
+	if(count == 1)
 		return textIndex;
-	else
-		return textIndex + 1;
+
+	return textIndex + 1;
 }
 
 void CGeneralTextHandler::dumpAllTexts()
 {
-	auto escapeString = [](std::string input)
-	{
-		boost::replace_all(input, "\\", "\\\\");
-		boost::replace_all(input, "\n", "\\n");
-		boost::replace_all(input, "\r", "\\r");
-		boost::replace_all(input, "\t", "\\t");
-		boost::replace_all(input, "\"", "\\\"");
-
-		return input;
-	};
-
 	logGlobal->info("BEGIN TEXT EXPORT");
 	for ( auto const & entry : stringsLocalizations)
-		if (stringsOverrides.count(entry.first) == 0)
-			logGlobal->info("\"%s\" : \"%s\",", entry.first, escapeString(entry.second));
-
-	for ( auto const & entry : stringsOverrides)
-		logGlobal->info("\"%s\" : \"%s\",", entry.first, escapeString(entry.second));
+	{
+		if (!entry.second.overrideValue.empty())
+			logGlobal->info(R"("%s" : "%s",)", entry.first, TextOperations::escapeString(entry.second.overrideValue));
+		else
+			logGlobal->info(R"("%s" : "%s",)", entry.first, TextOperations::escapeString(entry.second.baseValue));
+	}
 
 	logGlobal->info("END TEXT EXPORT");
 }
@@ -649,11 +582,23 @@ size_t CGeneralTextHandler::getCampaignLength(size_t campaignID) const
 	return 0;
 }
 
-std::string CGeneralTextHandler::getInstalledLanguage()
+std::string CGeneralTextHandler::getModLanguage(const std::string & modContext)
+{
+	if (modContext == "core")
+		return getInstalledLanguage();
+	return VLC->modh->getModLanguage(modContext);
+}
+
+std::string CGeneralTextHandler::getPreferredLanguage()
 {
 	return settings["general"]["language"].String();
 }
 
+std::string CGeneralTextHandler::getInstalledLanguage()
+{
+	return settings["session"]["language"].String();
+}
+
 std::string CGeneralTextHandler::getInstalledEncoding()
 {
 	auto explicitSetting = settings["general"]["encoding"].String();

lib/CGeneralTextHandler.h

@@ -9,52 +9,20 @@
  */
 #pragma once
 
-#include "JsonNode.h"
-
 VCMI_LIB_NAMESPACE_BEGIN
 
-/// Namespace that provides utilites for unicode support (UTF-8)
-namespace Unicode
-{
-	/// evaluates size of UTF-8 character
-	size_t DLL_LINKAGE getCharacterSize(char firstByte);
-
-	/// test if character is a valid UTF-8 symbol
-	/// maxSize - maximum number of bytes this symbol may consist from ( = remainer of string)
-	bool DLL_LINKAGE isValidCharacter(const char * character, size_t maxSize);
-
-	/// test if text contains ASCII-string (no need for unicode conversion)
-	bool DLL_LINKAGE isValidASCII(const std::string & text);
-	bool DLL_LINKAGE isValidASCII(const char * data, size_t size);
-
-	/// test if text contains valid UTF-8 sequence
-	bool DLL_LINKAGE isValidString(const std::string & text);
-	bool DLL_LINKAGE isValidString(const char * data, size_t size);
-
-	/// converts text to unicode from specified encoding or from one specified in settings
-	std::string DLL_LINKAGE toUnicode(const std::string & text);
-	std::string DLL_LINKAGE toUnicode(const std::string & text, const std::string & encoding);
-
-	/// converts text from unicode to specified encoding or to one specified in settings
-	/// NOTE: usage of these functions should be avoided if possible
-	std::string DLL_LINKAGE fromUnicode(const std::string & text);
-	std::string DLL_LINKAGE fromUnicode(const std::string & text, const std::string & encoding);
-
-	///delete (amount) UTF characters from right
-	DLL_LINKAGE void trimRight(std::string & text, const size_t amount = 1);
-};
-
 class CInputStream;
+class JsonNode;
 
 /// Parser for any text files from H3
 class DLL_LINKAGE CLegacyConfigParser
 {
+	std::string fileEncoding;
+
 	std::unique_ptr<char[]> data;
 	char * curr;
 	char * end;
 
-	void init(const std::unique_ptr<CInputStream> & input);
-
 	/// extracts part of quoted string.
 	std::string extractQuotedPart();
 
@@ -87,8 +55,9 @@ public:
 	/// end current line
 	bool endLine();
 
-	CLegacyConfigParser(std::string URI);
-	CLegacyConfigParser(const std::unique_ptr<CInputStream> & input);
+	explicit CLegacyConfigParser(std::string URI);
+private:
+	explicit CLegacyConfigParser(const std::unique_ptr<CInputStream> & input);
 };
 
 class CGeneralTextHandler;
@@ -146,30 +115,49 @@ public:
 /// Handles all text-related data in game
 class DLL_LINKAGE CGeneralTextHandler
 {
-	/// map identifier -> localization
-	std::unordered_map<std::string, std::string> stringsLocalizations;
+	struct StringState
+	{
+		/// Human-readable string that was added on registration
+		std::string baseValue;
+
+		/// Language of base string
+		std::string baseLanguage;
+
+		/// Translated human-readable string
+		std::string overrideValue;
+
+		/// Language of the override string
+		std::string overrideLanguage;
+
+		/// ID of mod that created this string
+		std::string modContext;
+	};
 
-	/// map identifier -> localization, high-priority strings from translation json
-	std::unordered_map<std::string, std::string> stringsOverrides;
+	/// map identifier -> localization
+	std::unordered_map<std::string, StringState> stringsLocalizations;
 
 	void readToVector(const std::string & sourceID, const std::string & sourceName);
 
 	/// number of scenarios in specific campaign. TODO: move to a better location
 	std::vector<size_t> scenariosCountPerCampaign;
 
-	/// Attempts to detect encoding & language of H3 files
-	void detectInstallParameters() const;
+	std::string getModLanguage(const std::string & modContext);
 public:
 
+	/// validates translation of specified language for specified mod
+	/// returns true if localization is valid and complete
+	/// any error messages will be written to log file
+	bool validateTranslation(const std::string & language, const std::string & modContext, JsonNode const & file) const;
+
 	/// Loads translation from provided json
 	/// Any entries loaded by this will have priority over texts registered normally
-	void loadTranslationOverrides(JsonNode const & file);
+	void loadTranslationOverrides(const std::string & language, const std::string & modContext, JsonNode const & file);
 
 	/// add selected string to internal storage
-	void registerString(const TextIdentifier & UID, const std::string & localized);
+	void registerString(const std::string & modContext, const TextIdentifier & UID, const std::string & localized);
 
 	/// add selected string to internal storage as high-priority strings
-	void registerStringOverride(const TextIdentifier & UID, const std::string & localized);
+	void registerStringOverride(const std::string & modContext, const std::string & language, const TextIdentifier & UID, const std::string & localized);
 
 	// returns true if identifier with such name was registered, even if not translated to current language
 	// not required right now, can be added if necessary
@@ -226,7 +214,7 @@ public:
 
 	std::vector<std::string> findStringsWithPrefix(std::string const & prefix);
 
-	int32_t pluralText(const int32_t textIndex, const int32_t count) const;
+	int32_t pluralText(int32_t textIndex, int32_t count) const;
 
 	size_t getCampaignLength(size_t campaignID) const;
 
@@ -234,7 +222,13 @@ public:
 	CGeneralTextHandler(const CGeneralTextHandler&) = delete;
 	CGeneralTextHandler operator=(const CGeneralTextHandler&) = delete;
 
+	/// Attempts to detect encoding & language of H3 files
+	static void detectInstallParameters();
+
 	/// Returns name of language preferred by user
+	static std::string getPreferredLanguage();
+
+	/// Returns name of language of Heroes III text files
 	static std::string getInstalledLanguage();
 
 	/// Returns name of encoding of Heroes III text files

lib/CHeroHandler.cpp

@@ -246,7 +246,7 @@ CHeroClass * CHeroClassHandler::loadFromJson(const std::string & scope, const Js
 	heroClass->imageMapFemale    = node["animation"]["map"]["female"].String();
 	heroClass->imageMapMale      = node["animation"]["map"]["male"].String();
 
-	VLC->generaltexth->registerString( heroClass->getNameTextID(), node["name"].String());
+	VLC->generaltexth->registerString(scope, heroClass->getNameTextID(), node["name"].String());
 
 	heroClass->affinity = vstd::find_pos(affinityStr, node["affinity"].String());
 
@@ -421,11 +421,11 @@ CHero * CHeroHandler::loadFromJson(const std::string & scope, const JsonNode & n
 	hero->sex = node["female"].Bool();
 	hero->special = node["special"].Bool();
 
-	VLC->generaltexth->registerString( hero->getNameTextID(), node["texts"]["name"].String());
-	VLC->generaltexth->registerString( hero->getBiographyTextID(), node["texts"]["biography"].String());
-	VLC->generaltexth->registerString( hero->getSpecialtyNameTextID(), node["texts"]["specialty"]["name"].String());
-	VLC->generaltexth->registerString( hero->getSpecialtyTooltipTextID(), node["texts"]["specialty"]["tooltip"].String());
-	VLC->generaltexth->registerString( hero->getSpecialtyDescriptionTextID(), node["texts"]["specialty"]["description"].String());
+	VLC->generaltexth->registerString(scope, hero->getNameTextID(), node["texts"]["name"].String());
+	VLC->generaltexth->registerString(scope, hero->getBiographyTextID(), node["texts"]["biography"].String());
+	VLC->generaltexth->registerString(scope, hero->getSpecialtyNameTextID(), node["texts"]["specialty"]["name"].String());
+	VLC->generaltexth->registerString(scope, hero->getSpecialtyTooltipTextID(), node["texts"]["specialty"]["tooltip"].String());
+	VLC->generaltexth->registerString(scope, hero->getSpecialtyDescriptionTextID(), node["texts"]["specialty"]["description"].String());
 
 	hero->iconSpecSmall = node["images"]["specialtySmall"].String();
 	hero->iconSpecLarge = node["images"]["specialtyLarge"].String();

lib/CModHandler.cpp

@@ -25,6 +25,7 @@
 #include "spells/CSpellHandler.h"
 #include "CSkillHandler.h"
 #include "CGeneralTextHandler.h"
+#include "Languages.h"
 #include "ScriptHandler.h"
 #include "RoadHandler.h"
 #include "RiverHandler.h"
@@ -216,7 +217,7 @@ std::vector<CIdentifierStorage::ObjectData> CIdentifierStorage::getPossibleIdent
 		// special scope that should have access to all in-game objects
 		if (request.localScope == CModHandler::scopeGame())
 		{
-			for (auto const & modName : VLC->modh->getActiveMods())
+			for(const auto & modName : VLC->modh->getActiveMods())
 				allowedScopes.insert(modName);
 		}
 
@@ -638,6 +639,12 @@ CModInfo::CModInfo(std::string identifier,const JsonNode & local, const JsonNode
 		vcmiCompatibleMin = Version::fromString(config["compatibility"]["min"].String());
 		vcmiCompatibleMax = Version::fromString(config["compatibility"]["max"].String());
 	}
+
+	if (!config["language"].isNull())
+		baseLanguage = config["language"].String();
+	else
+		baseLanguage = "english";
+
 	loadLocalData(local);
 }
 
@@ -936,8 +943,9 @@ std::vector<std::string> CModHandler::getModList(std::string path)
 
 bool CModHandler::isScopeReserved(const TModID & scope)
 {
-	static const std::array<TModID, 3> reservedScopes = {
-		"core", "map", "game"
+	//following scopes are reserved - either in use by mod system or by filesystem
+	static const std::array<TModID, 9> reservedScopes = {
+		"core", "map", "game", "root", "saves", "config", "local", "initial", "mapEditor"
 	};
 
 	return std::find(reservedScopes.begin(), reservedScopes.end(), scope) != reservedScopes.end();
@@ -1090,7 +1098,29 @@ void CModHandler::loadModFilesystems()
 	}
 }
 
-std::set<TModID> CModHandler::getModDependencies(TModID modId, bool & isModFound)
+TModID CModHandler::findResourceOrigin(const ResourceID & name)
+{
+	for(const auto & modID : boost::adaptors::reverse(activeMods))
+	{
+		if(CResourceHandler::get(modID)->existsResource(name))
+			return modID;
+	}
+
+	if(CResourceHandler::get("core")->existsResource(name))
+		return "core";
+
+	assert(0);
+	return "";
+}
+
+std::string CModHandler::getModLanguage(const TModID& modId) const
+{
+	if ( modId == "core")
+		return VLC->generaltexth->getInstalledLanguage();
+	return allMods.at(modId).baseLanguage;
+}
+
+std::set<TModID> CModHandler::getModDependencies(TModID modId, bool & isModFound) const
 {
 	auto it = allMods.find(modId);
 	isModFound = (it != allMods.end());
@@ -1099,7 +1129,7 @@ std::set<TModID> CModHandler::getModDependencies(TModID modId, bool & isModFound
 		return it->second.dependencies;
 
 	logMod->error("Mod not found: '%s'", modId);
-	return std::set<TModID>();
+	return {};
 }
 
 void CModHandler::initializeConfig()
@@ -1107,21 +1137,54 @@ void CModHandler::initializeConfig()
 	loadConfigFromFile("defaultMods.json");
 }
 
+bool CModHandler::validateTranslations(TModID modName) const
+{
+	bool result = true;
+	const auto & mod = allMods.at(modName);
+
+	{
+		auto fileList = mod.config["translations"].convertTo<std::vector<std::string> >();
+		JsonNode json = JsonUtils::assembleFromFiles(fileList);
+		result |= VLC->generaltexth->validateTranslation(mod.baseLanguage, modName, json);
+	}
+
+	for(const auto & language : Languages::getLanguageList())
+	{
+		if (!language.hasTranslation)
+			continue;
+
+		if (mod.config[language.identifier].isNull())
+			continue;
+
+		auto fileList = mod.config[language.identifier]["translations"].convertTo<std::vector<std::string> >();
+		JsonNode json = JsonUtils::assembleFromFiles(fileList);
+		result |= VLC->generaltexth->validateTranslation(language.identifier, modName, json);
+	}
+
+	return result;
+}
+
 void CModHandler::loadTranslation(TModID modName)
 {
-	auto const & mod = allMods[modName];
-	std::string language = VLC->generaltexth->getInstalledLanguage();
+	const auto & mod = allMods[modName];
+
+	std::string preferredLanguage = VLC->generaltexth->getPreferredLanguage();
+	std::string modBaseLanguage = allMods[modName].baseLanguage;
 
-	for (auto const & config : mod.config["translations"].Vector())
-		VLC->generaltexth->loadTranslationOverrides(JsonNode(ResourceID(config.String(), EResType::TEXT)));
+	auto baseTranslationList = mod.config["translations"].convertTo<std::vector<std::string> >();
+	auto extraTranslationList = mod.config[preferredLanguage]["translations"].convertTo<std::vector<std::string> >();
 
-	for (auto const & config : mod.config[language]["translations"].Vector())
-		VLC->generaltexth->loadTranslationOverrides(JsonNode(ResourceID(config.String(), EResType::TEXT)));
+	JsonNode baseTranslation = JsonUtils::assembleFromFiles(baseTranslationList);
+	JsonNode extraTranslation = JsonUtils::assembleFromFiles(extraTranslationList);
+
+	VLC->generaltexth->loadTranslationOverrides(modBaseLanguage, modName, baseTranslation);
+	VLC->generaltexth->loadTranslationOverrides(preferredLanguage, modName, extraTranslation);
 }
 
 void CModHandler::load()
 {
-	CStopWatch totalTime, timer;
+	CStopWatch totalTime;
+	CStopWatch timer;
 
 	logMod->info("\tInitializing content handler: %d ms", timer.getDiff());
 
@@ -1144,15 +1207,19 @@ void CModHandler::load()
 	for(const TModID & modName : activeMods)
 		content->load(allMods[modName]);
 
-	for(const TModID & modName : activeMods)
-		loadTranslation(modName);
-
 #if SCRIPTING_ENABLED
 	VLC->scriptHandler->performRegistration(VLC);//todo: this should be done before any other handlers load
 #endif
 
 	content->loadCustom();
 
+	for(const TModID & modName : activeMods)
+		loadTranslation(modName);
+
+	for(const TModID & modName : activeMods)
+		if (!validateTranslations(modName))
+			allMods[modName].validation = CModInfo::FAILED;
+
 	logMod->info("\tLoading mod data: %d ms", timer.getDiff());
 
 	VLC->creh->loadCrExpBon();

lib/CModHandler.h

@@ -219,6 +219,9 @@ public:
 	
 	/// version of the mod
 	Version version;
+
+	/// Base language of mod, all mod strings are assumed to be in this language
+	std::string baseLanguage;
 	
 	/// vcmi versions compatible with the mod
 
@@ -283,6 +286,8 @@ class DLL_LINKAGE CModHandler
 	void loadMods(std::string path, std::string parent, const JsonNode & modSettings, bool enableMods);
 	void loadOneMod(std::string modName, std::string parent, const JsonNode & modSettings, bool enableMods);
 	void loadTranslation(TModID modName);
+
+	bool validateTranslations(TModID modName) const;
 public:
 
 	/// returns true if scope is reserved for internal use and can not be used by mods
@@ -334,7 +339,12 @@ public:
 	void loadMods(bool onlyEssential = false);
 	void loadModFilesystems();
 
-	std::set<TModID> getModDependencies(TModID modId, bool & isModFound);
+	/// returns ID of mod that provides selected file resource
+	TModID findResourceOrigin(const ResourceID & name);
+
+	std::string getModLanguage(const TModID& modId) const;
+
+	std::set<TModID> getModDependencies(TModID modId, bool & isModFound) const;
 
 	/// returns list of all (active) mods
 	std::vector<std::string> getAllMods();

lib/CSkillHandler.cpp

@@ -211,7 +211,7 @@ CSkill * CSkillHandler::loadFromJson(const std::string & scope, const JsonNode &
 	CSkill * skill = new CSkill(SecondarySkill((si32)index), identifier);
 	skill->modScope = scope;
 
-	VLC->generaltexth->registerString(skill->getNameTextID(), json["name"].String());
+	VLC->generaltexth->registerString(scope, skill->getNameTextID(), json["name"].String());
 	switch(json["gainChance"].getType())
 	{
 	case JsonNode::JsonType::DATA_INTEGER:
@@ -236,7 +236,7 @@ CSkill * CSkillHandler::loadFromJson(const std::string & scope, const JsonNode &
 			skill->addNewBonus(bonus, level);
 		}
 		CSkill::LevelInfo & skillAtLevel = skill->at(level);
-		VLC->generaltexth->registerString(skill->getDescriptionTextID(level), levelNode["description"].String());
+		VLC->generaltexth->registerString(scope, skill->getDescriptionTextID(level), levelNode["description"].String());
 		skillAtLevel.iconSmall = levelNode["images"]["small"].String();
 		skillAtLevel.iconMedium = levelNode["images"]["medium"].String();
 		skillAtLevel.iconLarge = levelNode["images"]["large"].String();

lib/CTownHandler.cpp

@@ -604,8 +604,8 @@ void CTownHandler::loadBuilding(CTown * town, const std::string & stringID, cons
 	ret->modScope = source.meta;
 	ret->town = town;
 
-	VLC->generaltexth->registerString(ret->getNameTextID(), source["name"].String());
-	VLC->generaltexth->registerString(ret->getDescriptionTextID(), source["description"].String());
+	VLC->generaltexth->registerString(source.meta, ret->getNameTextID(), source["name"].String());
+	VLC->generaltexth->registerString(source.meta, ret->getDescriptionTextID(), source["description"].String());
 
 	ret->resources = TResources(source["cost"]);
 	ret->produce =   TResources(source["produce"]);
@@ -890,7 +890,7 @@ void CTownHandler::loadTown(CTown * town, const JsonNode & source)
 	town->namesCount = 0;
 	for (auto const & name : source["names"].Vector())
 	{
-		VLC->generaltexth->registerString(town->getRandomNameTextID(town->namesCount), name.String());
+		VLC->generaltexth->registerString(town->faction->modScope, town->getRandomNameTextID(town->namesCount), name.String());
 		town->namesCount += 1;
 	}
 
@@ -990,7 +990,7 @@ CFaction * CTownHandler::loadFromJson(const std::string & scope, const JsonNode
 	faction->modScope = scope;
 	faction->identifier = identifier;
 
-	VLC->generaltexth->registerString(faction->getNameTextID(), source["name"].String());
+	VLC->generaltexth->registerString(scope, faction->getNameTextID(), source["name"].String());
 
 	faction->creatureBg120 = source["creatureBackground"]["120px"].String();
 	faction->creatureBg130 = source["creatureBackground"]["130px"].String();

lib/GameConstants.h

@@ -792,6 +792,7 @@ public:
 		SCHOOL_OF_MAGIC = 47,
 		MAGIC_SPRING = 48,
 		MAGIC_WELL = 49,
+		MARKET_OF_TIME = 50,
 		MERCENARY_CAMP = 51,
 		MERMAID = 52,
 		MINE = 53,

lib/JsonDetail.cpp

@@ -12,7 +12,7 @@
 #include "JsonDetail.h"
 
 #include "VCMI_Lib.h"
-#include "CGeneralTextHandler.h"
+#include "TextOperations.h"
 #include "CModHandler.h"
 
 #include "filesystem/Filesystem.h"
@@ -169,7 +169,7 @@ JsonNode JsonParser::parse(std::string fileName)
 	}
 	else
 	{
-		if (!Unicode::isValidString(&input[0], input.size()))
+		if (!TextOperations::isValidUnicodeString(&input[0], input.size()))
 			error("Not a valid UTF-8 file", false);
 
 		extractValue(root);

lib/Languages.h

@@ -0,0 +1,81 @@
+/*
+ * Languages.h, part of VCMI engine
+ *
+ * Authors: listed in file AUTHORS in main folder
+ *
+ * License: GNU General Public License v2.0 or later
+ * Full text of license available in license.txt file, in main folder
+ *
+ */
+#pragma once
+
+namespace Languages
+{
+
+enum class ELanguages
+{
+	ENGLISH,
+	FRENCH,
+	GERMAN,
+	POLISH,
+	RUSSIAN,
+	UKRAINIAN,
+
+	COUNT
+};
+
+struct Options
+{
+	/// string identifier (ascii, lower-case), e.g. "english"
+	std::string identifier;
+
+	/// human-readable name of language in English
+	std::string nameEnglish;
+
+	/// human-readable name of language in its own language
+	std::string nameNative;
+
+	/// encoding that is used by H3 for this language
+	std::string encoding;
+
+	/// VCMI is capable of detecting H3 install in this language
+	bool hasDetection = false;
+
+	/// VCMI supports translations into this language
+	bool hasTranslation = false;
+};
+
+inline auto const & getLanguageList( )
+{
+	static const std::array<Options, 6> languages
+	{ {
+		{ "english",   "English",   "English",    "CP1252", true,  true },
+		{ "french",    "French",    "Français",   "CP1252", true, false },
+		{ "german",    "German",    "Deutsch",    "CP1252", true,  true },
+		{ "polish",    "Polish",    "Polski",     "CP1250", true,  true },
+		{ "russian",   "Russian",   "Русский",    "CP1251", true,  true },
+		{ "ukrainian", "Ukrainian", "Українська", "CP1251", true,  true }
+	} };
+	static_assert (languages.size() == static_cast<size_t>(ELanguages::COUNT), "Languages array is missing a value!" );
+
+	return languages;
+}
+
+inline const Options & getLanguageOptions( ELanguages language )
+{
+	assert(language < ELanguages::COUNT);
+	return getLanguageList()[static_cast<size_t>(language)];
+}
+
+inline const Options & getLanguageOptions( std::string language )
+{
+	for (auto const & entry : getLanguageList())
+		if (entry.identifier == language)
+			return entry;
+
+	static const Options emptyValue;
+	assert(0);
+	return emptyValue;
+}
+
+}

lib/NetPacksLib.cpp

@@ -28,6 +28,8 @@
 #include "StartInfo.h"
 #include "CPlayerState.h"
 #include "TerrainHandler.h"
+#include "mapping/CCampaignHandler.h"
+
 
 VCMI_LIB_NAMESPACE_BEGIN
 

lib/RiverHandler.cpp

@@ -18,6 +18,8 @@ VCMI_LIB_NAMESPACE_BEGIN
 RiverTypeHandler::RiverTypeHandler()
 {
 	objects.push_back(new RiverType);
+
+	VLC->generaltexth->registerString("core", objects[0]->getNameTextID(), "");
 }
 
 RiverType * RiverTypeHandler::loadFromJson(
@@ -31,16 +33,13 @@ RiverType * RiverTypeHandler::loadFromJson(
 	RiverType * info = new RiverType;
 
 	info->id              = RiverId(index);
-	if (identifier.find(':') == std::string::npos)
-		info->identifier = scope + ":" + identifier;
-	else
-		info->identifier = identifier;
-
+	info->identifier      = identifier;
+	info->modScope        = scope;
 	info->tilesFilename   = json["tilesFilename"].String();
 	info->shortIdentifier = json["shortIdentifier"].String();
 	info->deltaName       = json["delta"].String();
 
-	VLC->generaltexth->registerString(info->getNameTextID(), json["text"].String());
+	VLC->generaltexth->registerString(scope, info->getNameTextID(), json["text"].String());
 
 	return info;
 }
@@ -62,9 +61,14 @@ std::vector<bool> RiverTypeHandler::getDefaultAllowed() const
 	return {};
 }
 
+std::string RiverType::getJsonKey() const
+{
+	return modScope + ":" + identifier;
+}
+
 std::string RiverType::getNameTextID() const
 {
-	return TextIdentifier( "river", identifier,  "name" ).get();
+	return TextIdentifier( "river", modScope, identifier, "name" ).get();
 }
 
 std::string RiverType::getNameTranslated() const
@@ -74,7 +78,8 @@ std::string RiverType::getNameTranslated() const
 
 RiverType::RiverType():
 	id(River::NO_RIVER),
-	identifier("core:empty")
+	identifier("empty"),
+	modScope("core")
 {}
 
 VCMI_LIB_NAMESPACE_END

lib/RiverHandler.h

@@ -21,12 +21,13 @@ class DLL_LINKAGE RiverType : public EntityT<RiverId>
 {
 	friend class RiverTypeHandler;
 	std::string identifier;
+	std::string modScope;
 	RiverId id;
 
 public:
 	int32_t getIndex() const override { return id.getNum(); }
 	int32_t getIconIndex() const override { return 0; }
-	std::string getJsonKey() const override { return identifier;}
+	std::string getJsonKey() const override;
 	void registerIcons(const IconRegistar & cb) const override {}
 	RiverId getId() const override { return id;}
 	void updateFrom(const JsonNode & data) {};
@@ -44,6 +45,7 @@ public:
 	{
 		h & tilesFilename;
 		h & identifier;
+		h & modScope;
 		h & deltaName;
 		h & id;
 	}

lib/RoadHandler.cpp

@@ -18,6 +18,8 @@ VCMI_LIB_NAMESPACE_BEGIN
 RoadTypeHandler::RoadTypeHandler()
 {
 	objects.push_back(new RoadType);
+
+	VLC->generaltexth->registerString("core", objects[0]->getNameTextID(), "");
 }
 
 RoadType * RoadTypeHandler::loadFromJson(
@@ -31,16 +33,13 @@ RoadType * RoadTypeHandler::loadFromJson(
 	RoadType * info = new RoadType;
 
 	info->id              = RoadId(index);
-	if (identifier.find(':') == std::string::npos)
-		info->identifier = scope + ":" + identifier;
-	else
-		info->identifier = identifier;
-
+	info->identifier      = identifier;
+	info->modScope        = scope;
 	info->tilesFilename   = json["tilesFilename"].String();
 	info->shortIdentifier = json["shortIdentifier"].String();
 	info->movementCost    = json["moveCost"].Integer();
 
-	VLC->generaltexth->registerString(info->getNameTextID(), json["text"].String());
+	VLC->generaltexth->registerString(scope,info->getNameTextID(), json["text"].String());
 
 	return info;
 }
@@ -62,9 +61,14 @@ std::vector<bool> RoadTypeHandler::getDefaultAllowed() const
 	return {};
 }
 
+std::string RoadType::getJsonKey() const
+{
+	return modScope + ":" + identifier;
+}
+
 std::string RoadType::getNameTextID() const
 {
-	return TextIdentifier( "road", identifier,  "name" ).get();
+	return TextIdentifier( "road", modScope, identifier, "name" ).get();
 }
 
 std::string RoadType::getNameTranslated() const
@@ -74,7 +78,8 @@ std::string RoadType::getNameTranslated() const
 
 RoadType::RoadType():
 	id(Road::NO_ROAD),
-	identifier("core:empty"),
+	identifier("empty"),
+	modScope("core"),
 	movementCost(GameConstants::BASE_MOVEMENT_COST)
 {}
 VCMI_LIB_NAMESPACE_END

lib/RoadHandler.h

@@ -21,12 +21,13 @@ class DLL_LINKAGE RoadType : public EntityT<RoadId>
 {
 	friend class RoadTypeHandler;
 	std::string identifier;
+	std::string modScope;
 	RoadId id;
 
 public:
 	int32_t getIndex() const override { return id.getNum(); }
 	int32_t getIconIndex() const override { return 0; }
-	std::string getJsonKey() const override { return identifier;}
+	std::string getJsonKey() const override;
 	void registerIcons(const IconRegistar & cb) const override {}
 	RoadId getId() const override { return id;}
 	void updateFrom(const JsonNode & data) {};
@@ -44,6 +45,7 @@ public:
 	{
 		h & tilesFilename;
 		h & identifier;
+		h & modScope;
 		h & id;
 		h & movementCost;
 	}

lib/StartInfo.cpp

@@ -13,6 +13,8 @@
 #include "CGeneralTextHandler.h"
 #include "rmg/CMapGenOptions.h"
 #include "mapping/CMapInfo.h"
+#include "mapping/CCampaignHandler.h"
+#include "mapping/CMap.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 

lib/TerrainHandler.cpp

@@ -22,12 +22,8 @@ TerrainType * TerrainTypeHandler::loadFromJson( const std::string & scope, const
 	TerrainType * info = new TerrainType;
 
 	info->id = TerrainId(index);
-
-	if (identifier.find(':') == std::string::npos)
-		info->identifier = scope + ":" + identifier;
-	else
-		info->identifier = identifier;
-
+	info->identifier = identifier;
+	info->modScope = scope;
 	info->moveCost = static_cast<int>(json["moveCost"].Integer());
 	info->musicFilename = json["music"].String();
 	info->tilesFilename = json["tiles"].String();
@@ -36,7 +32,7 @@ TerrainType * TerrainTypeHandler::loadFromJson( const std::string & scope, const
 	info->transitionRequired = json["transitionRequired"].Bool();
 	info->terrainViewPatterns = json["terrainViewPatterns"].String();
 
-	VLC->generaltexth->registerString(info->getNameTextID(), json["text"].String());
+	VLC->generaltexth->registerString(scope, info->getNameTextID(), json["text"].String());
 
 	const JsonVector & unblockedVec = json["minimapUnblocked"].Vector();
 	info->minimapUnblocked =
@@ -177,9 +173,14 @@ bool TerrainType::isTransitionRequired() const
 	return transitionRequired;
 }
 
+std::string TerrainType::getJsonKey() const
+{
+	return modScope + ":" + identifier;
+}
+
 std::string TerrainType::getNameTextID() const
 {
-	return TextIdentifier( "terrain", identifier,  "name" ).get();
+	return TextIdentifier( "terrain", modScope, identifier, "name" ).get();
 }
 
 std::string TerrainType::getNameTranslated() const

lib/TerrainHandler.h

@@ -22,13 +22,14 @@ class DLL_LINKAGE TerrainType : public EntityT<TerrainId>
 {
 	friend class TerrainTypeHandler;
 	std::string identifier;
+	std::string modScope;
 	TerrainId id;
 	ui8 passabilityType;
 
 public:
 	int32_t getIndex() const override { return id.getNum(); }
 	int32_t getIconIndex() const override { return 0; }
-	std::string getJsonKey() const override { return identifier;}
+	std::string getJsonKey() const override;
 	void registerIcons(const IconRegistar & cb) const override {}
 	TerrainId getId() const override { return id;}
 	void updateFrom(const JsonNode & data) {};
@@ -78,6 +79,7 @@ public:
 		h & prohibitTransitions;
 		h & minimapBlocked;
 		h & minimapUnblocked;
+		h & modScope;
 		h & identifier;
 		h & musicFilename;
 		h & tilesFilename;

lib/TextOperations.cpp

@@ -0,0 +1,207 @@
+/*
+ * TextOperations.cpp, part of VCMI engine
+ *
+ * Authors: listed in file AUTHORS in main folder
+ *
+ * License: GNU General Public License v2.0 or later
+ * Full text of license available in license.txt file, in main folder
+ *
+ */
+#include "StdInc.h"
+#include "TextOperations.h"
+
+#include "CGeneralTextHandler.h"
+
+#include <boost/locale.hpp>
+
+VCMI_LIB_NAMESPACE_BEGIN
+
+size_t TextOperations::getUnicodeCharacterSize(char firstByte)
+{
+	// length of utf-8 character can be determined from 1st byte by counting number of highest bits set to 1:
+	// 0xxxxxxx -> 1 -  ASCII chars
+	// 110xxxxx -> 2
+	// 1110xxxx -> 3
+	// 11110xxx -> 4 - last allowed in current standard
+
+	auto value = static_cast<uint8_t>(firstByte);
+
+	if ((value & 0b10000000) == 0)
+		return 1; // ASCII
+
+	if ((value & 0b11100000) == 0b11000000)
+		return 2;
+
+	if ((value & 0b11110000) == 0b11100000)
+		return 3;
+
+	if ((value & 0b11111000) == 0b11110000)
+		return 4;
+
+	assert(0);// invalid unicode sequence
+	return 4;
+}
+
+bool TextOperations::isValidUnicodeCharacter(const char * character, size_t maxSize)
+{
+	assert(maxSize > 0);
+
+	auto value = static_cast<uint8_t>(character[0]);
+
+	// ASCII
+	if ( value < 0b10000000)
+		return maxSize > 0;
+
+	// can't be first byte in UTF8
+	if (value < 0b11000000)
+		return false;
+
+	// above maximum allowed in standard (UTF codepoints are capped at 0x0010FFFF)
+	if (value > 0b11110000)
+		return false;
+
+	// first character must follow rules checked in getUnicodeCharacterSize
+	size_t size = getUnicodeCharacterSize(character[0]);
+
+	if (size > maxSize)
+		return false;
+
+	// remaining characters must have highest bit set to 1
+	for (size_t i = 1; i < size; i++)
+	{
+		auto characterValue = static_cast<uint8_t>(character[i]);
+		if (characterValue < 0b10000000)
+			return false;
+	}
+	return true;
+}
+
+bool TextOperations::isValidASCII(const std::string & text)
+{
+	for (const char & ch : text)
+		if (static_cast<uint8_t>(ch) >= 0x80 )
+			return false;
+	return true;
+}
+
+bool TextOperations::isValidASCII(const char * data, size_t size)
+{
+	for (size_t i=0; i<size; i++)
+		if (static_cast<uint8_t>(data[i]) >= 0x80 )
+			return false;
+	return true;
+}
+
+bool TextOperations::isValidUnicodeString(const std::string & text)
+{
+	for (size_t i=0; i<text.size(); i += getUnicodeCharacterSize(text[i]))
+	{
+		if (!isValidUnicodeCharacter(text.data() + i, text.size() - i))
+			return false;
+	}
+	return true;
+}
+
+bool TextOperations::isValidUnicodeString(const char * data, size_t size)
+{
+	for (size_t i=0; i<size; i += getUnicodeCharacterSize(data[i]))
+	{
+		if (!isValidUnicodeCharacter(data + i, size - i))
+			return false;
+	}
+	return true;
+}
+
+uint32_t TextOperations::getUnicodeCodepoint(const char * data, size_t maxSize)
+{
+	assert(isValidUnicodeCharacter(data, maxSize));
+	if (!isValidUnicodeCharacter(data, maxSize))
+		return 0;
+
+	// https://en.wikipedia.org/wiki/UTF-8#Encoding
+	switch (getUnicodeCharacterSize(data[0]))
+	{
+		case 1:
+			return static_cast<uint8_t>(data[0]) & 0b1111111;
+		case 2:
+			return
+				((static_cast<uint8_t>(data[0]) & 0b11111 ) << 6) +
+				((static_cast<uint8_t>(data[1]) & 0b111111) << 0) ;
+		case 3:
+			return
+				((static_cast<uint8_t>(data[0]) & 0b1111 )  << 12) +
+				((static_cast<uint8_t>(data[1]) & 0b111111) << 6) +
+				((static_cast<uint8_t>(data[2]) & 0b111111) << 0) ;
+		case 4:
+			return
+				((static_cast<uint8_t>(data[0]) & 0b111 )   << 18) +
+				((static_cast<uint8_t>(data[1]) & 0b111111) << 12) +
+				((static_cast<uint8_t>(data[2]) & 0b111111) << 6) +
+				((static_cast<uint8_t>(data[3]) & 0b111111) << 0) ;
+	}
+
+	assert(0);
+	return 0;
+}
+
+uint32_t TextOperations::getUnicodeCodepoint(char data, const std::string & encoding )
+{
+	std::string stringNative(1, data);
+	std::string stringUnicode = toUnicode(stringNative, encoding);
+
+	if (stringUnicode.empty())
+		return 0;
+
+	return getUnicodeCodepoint(stringUnicode.data(), stringUnicode.size());
+}
+
+std::string TextOperations::toUnicode(const std::string &text, const std::string &encoding)
+{
+	return boost::locale::conv::to_utf<char>(text, encoding);
+}
+
+std::string TextOperations::fromUnicode(const std::string &text, const std::string &encoding)
+{
+	return boost::locale::conv::from_utf<char>(text, encoding);
+}
+
+void TextOperations::trimRightUnicode(std::string & text, const size_t amount)
+{
+	if(text.empty())
+		return;
+	//todo: more efficient algorithm
+	for(int i = 0; i< amount; i++){
+		auto b = text.begin();
+		auto e = text.end();
+		size_t lastLen = 0;
+		size_t len = 0;
+		while (b != e) {
+			lastLen = len;
+			size_t n = getUnicodeCharacterSize(*b);
+
+			if(!isValidUnicodeCharacter(&(*b),e-b))
+			{
+				logGlobal->error("Invalid UTF8 sequence");
+				break;//invalid sequence will be trimmed
+			}
+
+			len += n;
+			b += n;
+		}
+
+		text.resize(lastLen);
+	}
+}
+
+std::string TextOperations::escapeString(std::string input)
+{
+	boost::replace_all(input, "\\", "\\\\");
+	boost::replace_all(input, "\n", "\\n");
+	boost::replace_all(input, "\r", "\\r");
+	boost::replace_all(input, "\t", "\\t");
+	boost::replace_all(input, "\"", "\\\"");
+
+	return input;
+}
+
+VCMI_LIB_NAMESPACE_END

lib/TextOperations.h

@@ -0,0 +1,80 @@
+/*
+ * TextOperations.h, part of VCMI engine
+ *
+ * Authors: listed in file AUTHORS in main folder
+ *
+ * License: GNU General Public License v2.0 or later
+ * Full text of license available in license.txt file, in main folder
+ *
+ */
+#pragma once
+
+VCMI_LIB_NAMESPACE_BEGIN
+
+/// Namespace that provides utilites for unicode support (UTF-8)
+namespace TextOperations
+{
+	/// returns 32-bit UTF codepoint for UTF-8 character symbol
+	uint32_t DLL_LINKAGE getUnicodeCodepoint(const char *data, size_t maxSize);
+
+	/// returns 32-bit UTF codepoint for character symbol in selected single-byte encoding
+	uint32_t DLL_LINKAGE getUnicodeCodepoint(char data, const std::string & encoding );
+
+	/// returns length (in bytes) of UTF-8 character starting from specified character
+	size_t DLL_LINKAGE getUnicodeCharacterSize(char firstByte);
+
+	/// test if character is a valid UTF-8 symbol
+	/// maxSize - maximum number of bytes this symbol may consist from ( = remainer of string)
+	bool DLL_LINKAGE isValidUnicodeCharacter(const char * character, size_t maxSize);
+
+	/// returns true if text contains valid ASCII-string
+	/// Note that since UTF-8 extends ASCII, any ASCII string is also UTF-8 string
+	bool DLL_LINKAGE isValidASCII(const std::string & text);
+	bool DLL_LINKAGE isValidASCII(const char * data, size_t size);
+
+	/// test if text contains valid UTF-8 sequence
+	bool DLL_LINKAGE isValidUnicodeString(const std::string & text);
+	bool DLL_LINKAGE isValidUnicodeString(const char * data, size_t size);
+
+	/// converts text to UTF-8 from specified encoding or from one specified in settings
+	std::string DLL_LINKAGE toUnicode(const std::string & text, const std::string & encoding);
+
+	/// converts text from unicode to specified encoding or to one specified in settings
+	/// NOTE: usage of these functions should be avoided if possible
+	std::string DLL_LINKAGE fromUnicode(const std::string & text, const std::string & encoding);
+
+	///delete specified amount of UTF-8 characters from right
+	DLL_LINKAGE void trimRightUnicode(std::string & text, size_t amount = 1);
+
+	/// converts number into string using metric system prefixes, e.g. 'k' or 'M' to keep resulting strings within specified size
+	/// Note that resulting string may have more symbols than digits: minus sign and prefix symbol
+	template<typename Arithmetic>
+	inline std::string formatMetric(Arithmetic number, int maxDigits);
+
+	/// replaces all symbols that normally need escaping with appropriate escape sequences
+	std::string escapeString(std::string input);
+};
+
+
+
+template<typename Arithmetic>
+inline std::string TextOperations::formatMetric(Arithmetic number, int maxDigits)
+{
+	Arithmetic max = std::pow(10, maxDigits);
+	if (std::abs(number) < max)
+		return std::to_string(number);
+
+	std::string symbols = " kMGTPE";
+	auto iter = symbols.begin();
+
+	while (std::abs(number) >= max)
+	{
+		number /= 1000;
+		iter++;
+
+		assert(iter != symbols.end());//should be enough even for int64
+	}
+	return std::to_string(number) + *iter;
+}
+
+VCMI_LIB_NAMESPACE_END

lib/filesystem/CBinaryReader.cpp

@@ -11,7 +11,7 @@
 #include "CBinaryReader.h"
 
 #include "CInputStream.h"
-#include "../CGeneralTextHandler.h"
+#include "../TextOperations.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 
@@ -86,7 +86,7 @@ INSTANTIATE(si64, readInt64)
 
 #undef INSTANTIATE
 
-std::string CBinaryReader::readString()
+std::string CBinaryReader::readBaseString()
 {
 	unsigned int len = readUInt32();
 	assert(len <= 500000); //not too long
@@ -95,10 +95,7 @@ std::string CBinaryReader::readString()
 		std::string ret;
 		ret.resize(len);
 		read(reinterpret_cast<ui8*>(&ret[0]), len);
-		//FIXME: any need to move this into separate "read localized string" method?
-		if (Unicode::isValidASCII(ret))
-			return ret;
-		return Unicode::toUnicode(ret);
+		return ret;
 	}
 	return "";
 

lib/filesystem/CBinaryReader.h

@@ -73,7 +73,8 @@ public:
 	ui64 readUInt64();
 	si64 readInt64();
 
-	std::string readString();
+	/// Reads string without any encoding conversions
+	std::string readBaseString();
 
 	inline bool readBool()
 	{

lib/mapObjects/CGHeroInstance.cpp

@@ -987,11 +987,13 @@ void CGHeroInstance::initExp(CRandomGenerator & rand)
 
 std::string CGHeroInstance::nodeName() const
 {
-	return "Hero " + getNameTextID();
+	return "Hero " + getNameTranslated();
 }
 
 std::string CGHeroInstance::getNameTranslated() const
 {
+	if (!nameCustom.empty())
+		return nameCustom;
 	return VLC->generaltexth->translate(getNameTextID());
 }
 
@@ -1009,6 +1011,9 @@ std::string CGHeroInstance::getNameTextID() const
 
 std::string CGHeroInstance::getBiographyTranslated() const
 {
+	if (!biographyCustom.empty())
+		return biographyCustom;
+
 	return VLC->generaltexth->translate(getBiographyTextID());
 }
 

lib/mapObjects/CGHeroInstance.h

@@ -151,11 +151,13 @@ public:
 
 	//////////////////////////////////////////////////////////////////////////
 
+	std::string getBiographyTranslated() const;
 	std::string getNameTranslated() const;
-	std::string getNameTextID() const;
 
-	std::string getBiographyTranslated() const;
+private:
+	std::string getNameTextID() const;
 	std::string getBiographyTextID() const;
+public:
 
 	bool hasSpellbook() const;
 	int maxSpellLevel() const;

lib/mapObjects/CObjectClassesHandler.cpp

@@ -233,7 +233,7 @@ ObjectClass * CObjectClassesHandler::loadFromJson(const std::string & scope, con
 	obj->base = json["base"];
 	obj->id = index;
 
-	VLC->generaltexth->registerString(obj->getNameTextID(), json["name"].String());
+	VLC->generaltexth->registerString(scope, obj->getNameTextID(), json["name"].String());
 
 	obj->objects.resize(json["lastReservedIndex"].Float() + 1);
 

lib/mapObjects/CQuest.cpp

@@ -234,7 +234,7 @@ void CQuest::getVisitText(MetaString &iwText, std::vector<Component> &components
 			//FIXME: portrait may not match hero, if custom portrait was set in map editor
 			components.emplace_back(Component::HERO_PORTRAIT, VLC->heroh->objects[m13489val]->imageIndex, 0, 0);
 			if(!isCustom)
-				iwText.addReplacement(VLC->heroh->objects[m13489val]->getNameTextID());
+				iwText.addReplacement(VLC->heroh->objects[m13489val]->getNameTranslated());
 			break;
 		case MISSION_KILL_CREATURE:
 			{
@@ -373,7 +373,7 @@ void CQuest::getRolloverText(MetaString &ms, bool onHover) const
 			}
 			break;
 		case MISSION_HERO:
-			ms.addReplacement(VLC->heroh->objects[m13489val]->getNameTextID());
+			ms.addReplacement(VLC->heroh->objects[m13489val]->getNameTranslated());
 			break;
 		case MISSION_PLAYER:
 			ms.addReplacement(VLC->generaltexth->colors[m13489val]);
@@ -456,7 +456,7 @@ void CQuest::getCompletionText(MetaString &iwText, std::vector<Component> &compo
 			break;
 		case MISSION_HERO:
 			if (!isCustomComplete)
-				iwText.addReplacement(VLC->heroh->objects[m13489val]->getNameTextID());
+				iwText.addReplacement(VLC->heroh->objects[m13489val]->getNameTranslated());
 			break;
 		case MISSION_PLAYER:
 			if (!isCustomComplete)

lib/mapObjects/CommonConstructors.cpp

@@ -141,7 +141,7 @@ void CDwellingInstanceConstructor::initTypeData(const JsonNode & input)
 	if (input.Struct().count("name") == 0)
 		logMod->warn("Dwelling %s missing name!", getJsonKey());
 
-	VLC->generaltexth->registerString(getNameTextID(), input["name"].String());
+	VLC->generaltexth->registerString( input.meta, getNameTextID(), input["name"].String());
 
 	const JsonVector & levels = input["creatures"].Vector();
 	const auto totalLevels = levels.size();
@@ -267,7 +267,7 @@ void CBankInstanceConstructor::initTypeData(const JsonNode & input)
 	if (input.Struct().count("name") == 0)
 		logMod->warn("Bank %s missing name!", getJsonKey());
 
-	VLC->generaltexth->registerString(getNameTextID(), input["name"].String());
+	VLC->generaltexth->registerString(input.meta, getNameTextID(), input["name"].String());
 
 	levels = input["levels"].Vector();
 	bankResetDuration = static_cast<si32>(input["resetDuration"].Float());

lib/mapObjects/MiscObjects.cpp

@@ -1615,7 +1615,7 @@ std::string CGShrine::getHoverText(PlayerColor player) const
 	if(wasVisited(player))
 	{
 		hoverName += "\n" + VLC->generaltexth->allTexts[355]; // + (learn %s)
-		boost::algorithm::replace_first(hoverName,"%s", spell.toSpell()->getNameTextID());
+		boost::algorithm::replace_first(hoverName,"%s", spell.toSpell()->getNameTranslated());
 	}
 	return hoverName;
 }

lib/mapObjects/ObjectTemplate.cpp

@@ -198,7 +198,7 @@ void ObjectTemplate::readMsk()
 
 void ObjectTemplate::readMap(CBinaryReader & reader)
 {
-	animationFile = reader.readString();
+	animationFile = reader.readBaseString();
 
 	setSize(8, 6);
 	ui8 blockMask[6];

lib/mapping/CCampaignHandler.cpp

@@ -17,10 +17,13 @@
 #include "../VCMI_Lib.h"
 #include "../vcmi_endian.h"
 #include "../CGeneralTextHandler.h"
+#include "../TextOperations.h"
 #include "../StartInfo.h"
+#include "../CModHandler.h"
 #include "../CArtHandler.h" //for hero crossover
 #include "../mapObjects/CGHeroInstance.h"//for hero crossover
 #include "../CHeroHandler.h"
+#include "../Languages.h"
 #include "CMapService.h"
 #include "CMap.h"
 #include "CMapInfo.h"
@@ -40,31 +43,41 @@ bool CScenarioTravel::STravelBonus::isBonusForHero() const
 
 CCampaignHeader CCampaignHandler::getHeader( const std::string & name)
 {
-	std::vector<ui8> cmpgn = getFile(name, true)[0];
+	ResourceID resourceID(name, EResType::CAMPAIGN);
+	std::string modName = VLC->modh->findResourceOrigin(resourceID);
+	std::string language = VLC->modh->getModLanguage(modName);
+	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	auto fileStream = CResourceHandler::get(modName)->load(resourceID);
+
+	std::vector<ui8> cmpgn = getFile(std::move(fileStream), true)[0];
 
 	CMemoryStream stream(cmpgn.data(), cmpgn.size());
 	CBinaryReader reader(&stream);
-	CCampaignHeader ret = readHeaderFromMemory(reader);
-	ret.filename = name;
+	CCampaignHeader ret = readHeaderFromMemory(reader, name, modName, encoding);
 
 	return ret;
 }
 
 std::unique_ptr<CCampaign> CCampaignHandler::getCampaign( const std::string & name )
 {
+	ResourceID resourceID(name, EResType::CAMPAIGN);
+	std::string modName = VLC->modh->findResourceOrigin(resourceID);
+	std::string language = VLC->modh->getModLanguage(modName);
+	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	auto fileStream = CResourceHandler::get(modName)->load(resourceID);
+
 	auto ret = std::make_unique<CCampaign>();
 
-	std::vector<std::vector<ui8>> file = getFile(name, false);
+	std::vector<std::vector<ui8>> file = getFile(std::move(fileStream), false);
 
 	CMemoryStream stream(file[0].data(), file[0].size());
 	CBinaryReader reader(&stream);
-	ret->header = readHeaderFromMemory(reader);
-	ret->header.filename = name;
+	ret->header = readHeaderFromMemory(reader, name, modName, encoding);
 
 	int howManyScenarios = static_cast<int>(VLC->generaltexth->getCampaignLength(ret->header.mapVersion));
 	for(int g=0; g<howManyScenarios; ++g)
 	{
-		CCampaignScenario sc = readScenarioFromMemory(reader, ret->header.version, ret->header.mapVersion);
+		CCampaignScenario sc = readScenarioFromMemory(reader, encoding, ret->header.version, ret->header.mapVersion);
 		ret->scenarios.push_back(sc);
 	}
 
@@ -88,7 +101,9 @@ std::unique_ptr<CCampaign> CCampaignHandler::getCampaign( const std::string & na
 		auto hdr = mapService.loadMapHeader(
 			reinterpret_cast<const ui8 *>(ret->mapPieces[scenarioID].c_str()),
 			static_cast<int>(ret->mapPieces[scenarioID].size()),
-			scenarioName);
+			scenarioName,
+			modName,
+			encoding);
 		ret->scenarios[scenarioID].scenarioName = hdr->name;
 		scenarioID++;
 	}
@@ -110,23 +125,31 @@ std::unique_ptr<CCampaign> CCampaignHandler::getCampaign( const std::string & na
 	return ret;
 }
 
-CCampaignHeader CCampaignHandler::readHeaderFromMemory( CBinaryReader & reader )
+std::string CCampaignHandler::readLocalizedString(CBinaryReader & reader, std::string encoding)
+{
+	return TextOperations::toUnicode(reader.readBaseString(), encoding);
+}
+
+CCampaignHeader CCampaignHandler::readHeaderFromMemory( CBinaryReader & reader, std::string filename, std::string modName, std::string encoding )
 {
 	CCampaignHeader ret;
 
 	ret.version = reader.readUInt32();
 	ret.mapVersion = reader.readUInt8() - 1;//change range of it from [1, 20] to [0, 19]
-	ret.name = reader.readString();
-	ret.description = reader.readString();
+	ret.name = readLocalizedString(reader, encoding);
+	ret.description = readLocalizedString(reader, encoding);
 	if (ret.version > CampaignVersion::RoE)
 		ret.difficultyChoosenByPlayer = reader.readInt8();
 	else
 		ret.difficultyChoosenByPlayer = 0;
 	ret.music = reader.readInt8();
+	ret.filename = filename;
+	ret.modName = modName;
+	ret.encoding = encoding;
 	return ret;
 }
 
-CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & reader, int version, int mapVersion )
+CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & reader, std::string encoding, int version, int mapVersion )
 {
 	auto prologEpilogReader = [&]() -> CCampaignScenario::SScenarioPrologEpilog
 	{
@@ -136,14 +159,14 @@ CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & read
 		{
 			ret.prologVideo = reader.readUInt8();
 			ret.prologMusic = reader.readUInt8();
-			ret.prologText = reader.readString();
+			ret.prologText = readLocalizedString(reader, encoding);
 		}
 		return ret;
 	};
 
 	CCampaignScenario ret;
 	ret.conquered = false;
-	ret.mapName = reader.readString();
+	ret.mapName = readLocalizedString(reader, encoding);
 	ret.packedMapSize = reader.readUInt32();
 	if(mapVersion == 18)//unholy alliance
 	{
@@ -155,7 +178,7 @@ CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & read
 	}
 	ret.regionColor = reader.readUInt8();
 	ret.difficulty = reader.readUInt8();
-	ret.regionText = reader.readString();
+	ret.regionText = readLocalizedString(reader, encoding);
 	ret.prolog = prologEpilogReader();
 	ret.epilog = prologEpilogReader();
 
@@ -305,9 +328,9 @@ CScenarioTravel CCampaignHandler::readScenarioTravelFromMemory(CBinaryReader & r
 	return ret;
 }
 
-std::vector< std::vector<ui8> > CCampaignHandler::getFile(const std::string & name, bool headerOnly)
+std::vector< std::vector<ui8> > CCampaignHandler::getFile(std::unique_ptr<CInputStream> file, bool headerOnly)
 {
-	CCompressedStream stream(CResourceHandler::get()->load(ResourceID(name, EResType::CAMPAIGN)), true);
+	CCompressedStream stream(std::move(file), true);
 
 	std::vector< std::vector<ui8> > ret;
 	do
@@ -452,7 +475,7 @@ CMap * CCampaignState::getMap(int scenarioId) const
 	std::string & mapContent = camp->mapPieces.find(scenarioId)->second;
 	const auto * buffer = reinterpret_cast<const ui8 *>(mapContent.data());
 	CMapService mapService;
-	return mapService.loadMap(buffer, static_cast<int>(mapContent.size()), scenarioName).release();
+	return mapService.loadMap(buffer, static_cast<int>(mapContent.size()), scenarioName, camp->header.modName, camp->header.encoding).release();
 }
 
 std::unique_ptr<CMapHeader> CCampaignState::getHeader(int scenarioId) const
@@ -466,7 +489,7 @@ std::unique_ptr<CMapHeader> CCampaignState::getHeader(int scenarioId) const
 	std::string & mapContent = camp->mapPieces.find(scenarioId)->second;
 	const auto * buffer = reinterpret_cast<const ui8 *>(mapContent.data());
 	CMapService mapService;
-	return mapService.loadMapHeader(buffer, static_cast<int>(mapContent.size()), scenarioName);
+	return mapService.loadMapHeader(buffer, static_cast<int>(mapContent.size()), scenarioName, camp->header.modName, camp->header.encoding);
 }
 
 std::shared_ptr<CMapInfo> CCampaignState::getMapInfo(int scenarioId) const
@@ -522,7 +545,4 @@ std::string CCampaignHandler::prologVoiceName(ui8 index)
 	return "";
 }
 
-
-
-
 VCMI_LIB_NAMESPACE_END

lib/mapping/CCampaignHandler.h

@@ -16,6 +16,7 @@ VCMI_LIB_NAMESPACE_BEGIN
 struct StartInfo;
 class CGHeroInstance;
 class CBinaryReader;
+class CInputStream;
 class CMap;
 class CMapHeader;
 class CMapInfo;
@@ -42,7 +43,8 @@ public:
 	ui8 music = 0; //CmpMusic.txt, start from 0
 
 	std::string filename;
-	ui8 loadFromLod = 0; //if true, this campaign must be loaded fro, .lod file
+	std::string modName;
+	std::string encoding;
 
 	template <typename Handler> void serialize(Handler &h, const int formatVersion)
 	{
@@ -53,7 +55,8 @@ public:
 		h & difficultyChoosenByPlayer;
 		h & music;
 		h & filename;
-		h & loadFromLod;
+		h & modName;
+		h & encoding;
 	}
 };
 
@@ -181,7 +184,7 @@ class DLL_LINKAGE CCampaignState
 {
 public:
 	std::unique_ptr<CCampaign> camp;
-	std::string campaignName;
+	std::string fileEncoding;
 	std::vector<ui8> mapsConquered, mapsRemaining;
 	boost::optional<si32> currentMap;
 
@@ -206,7 +209,6 @@ public:
 	template <typename Handler> void serialize(Handler &h, const int version)
 	{
 		h & camp;
-		h & campaignName;
 		h & mapsRemaining;
 		h & mapsConquered;
 		h & currentMap;
@@ -218,12 +220,15 @@ class DLL_LINKAGE CCampaignHandler
 {
 	std::vector<size_t> scenariosCountPerCampaign;
 
-	static CCampaignHeader readHeaderFromMemory(CBinaryReader & reader);
-	static CCampaignScenario readScenarioFromMemory(CBinaryReader & reader, int version, int mapVersion );
+	static std::string readLocalizedString(CBinaryReader & reader, std::string encoding);
+
+	static CCampaignHeader readHeaderFromMemory(CBinaryReader & reader, std::string filename, std::string modName, std::string encoding);
+	static CCampaignScenario readScenarioFromMemory(CBinaryReader & reader, std::string encoding, int version, int mapVersion );
 	static CScenarioTravel readScenarioTravelFromMemory(CBinaryReader & reader, int version);
 	/// returns h3c split in parts. 0 = h3c header, 1-end - maps (binary h3m)
 	/// headerOnly - only header will be decompressed, returned vector wont have any maps
-	static std::vector< std::vector<ui8> > getFile(const std::string & name, bool headerOnly);
+	static std::vector<std::vector<ui8>> getFile(std::unique_ptr<CInputStream> file, bool headerOnly);
+
 public:
 	static std::string prologVideoName(ui8 index);
 	static std::string prologMusicName(ui8 index);

lib/mapping/CMapInfo.cpp

@@ -14,6 +14,8 @@
 #include "../StartInfo.h"
 #include "../GameConstants.h"
 #include "CMapService.h"
+#include "CMap.h"
+#include "CCampaignHandler.h"
 
 #include "../filesystem/Filesystem.h"
 #include "../serializer/CMemorySerializer.h"

lib/mapping/CMapInfo.h

@@ -9,21 +9,14 @@
  */
 #pragma once
 
-// Forward class declarations aren't enough here. The compiler
-// generated CMapInfo d-tor, generates the std::unique_ptr d-tor as well here
-// as a inline method. The std::unique_ptr d-tor requires a complete type. Defining
-// the CMapInfo d-tor to let the compiler add the d-tor stuff in the .cpp file
-// would work with one exception. It prevents the generation of the move
-// constructor which is needed. (Writing such a c-tor is nasty.) With the
-// new c++11 keyword "default" for constructors this problem could be solved. But it isn't
-// available for Visual Studio for now. (Empty d-tor in .cpp would be required anyway)
-#include "CMap.h"
-#include "CCampaignHandler.h"
-
 VCMI_LIB_NAMESPACE_BEGIN
 
 struct StartInfo;
 
+class CMapHeader;
+class CCampaignHeader;
+class ResourceID;
+
 /**
  * A class which stores the count of human players and all players, the filename,
  * scenario options, the map header information,...
@@ -44,7 +37,11 @@ public:
 	CMapInfo();
 	virtual ~CMapInfo();
 
-	CMapInfo &operator=(CMapInfo &&other);
+	CMapInfo(CMapInfo &&other) = delete;
+	CMapInfo(const CMapInfo &other) = delete;
+
+	CMapInfo &operator=(CMapInfo &&other) = delete;
+	CMapInfo &operator=(const CMapInfo &other) = delete;
 
 	void mapInit(const std::string & fname);
 	void saveInit(const ResourceID & file);

lib/mapping/CMapService.cpp

@@ -15,6 +15,8 @@
 #include "../filesystem/CCompressedStream.h"
 #include "../filesystem/CMemoryStream.h"
 #include "../filesystem/CMemoryBuffer.h"
+#include "../CModHandler.h"
+#include "../Languages.h"
 
 #include "CMap.h"
 
@@ -26,20 +28,28 @@ VCMI_LIB_NAMESPACE_BEGIN
 
 std::unique_ptr<CMap> CMapService::loadMap(const ResourceID & name) const
 {
+	std::string modName = VLC->modh->findResourceOrigin(name);
+	std::string language = VLC->modh->getModLanguage(modName);
+	std::string encoding = Languages::getLanguageOptions(language).encoding;
+
 	auto stream = getStreamFromFS(name);
-	return getMapLoader(stream)->loadMap();
+	return getMapLoader(stream, name.getName(), modName, encoding)->loadMap();
 }
 
 std::unique_ptr<CMapHeader> CMapService::loadMapHeader(const ResourceID & name) const
 {
+	std::string modName = VLC->modh->findResourceOrigin(name);
+	std::string language = VLC->modh->getModLanguage(modName);
+	std::string encoding = Languages::getLanguageOptions(language).encoding;
+
 	auto stream = getStreamFromFS(name);
-	return getMapLoader(stream)->loadMapHeader();
+	return getMapLoader(stream, name.getName(), modName, encoding)->loadMapHeader();
 }
 
-std::unique_ptr<CMap> CMapService::loadMap(const ui8 * buffer, int size, const std::string & name) const
+std::unique_ptr<CMap> CMapService::loadMap(const ui8 * buffer, int size, const std::string & name,  const std::string & modName, const std::string & encoding) const
 {
 	auto stream = getStreamFromMem(buffer, size);
-	std::unique_ptr<CMap> map(getMapLoader(stream)->loadMap());
+	std::unique_ptr<CMap> map(getMapLoader(stream, name, modName, encoding)->loadMap());
 	std::unique_ptr<CMapHeader> header(map.get());
 
 	//might be original campaign and require patch
@@ -49,10 +59,10 @@ std::unique_ptr<CMap> CMapService::loadMap(const ui8 * buffer, int size, const s
 	return map;
 }
 
-std::unique_ptr<CMapHeader> CMapService::loadMapHeader(const ui8 * buffer, int size, const std::string & name) const
+std::unique_ptr<CMapHeader> CMapService::loadMapHeader(const ui8 * buffer, int size, const std::string & name, const std::string & modName, const std::string & encoding) const
 {
 	auto stream = getStreamFromMem(buffer, size);
-	std::unique_ptr<CMapHeader> header = getMapLoader(stream)->loadMapHeader();
+	std::unique_ptr<CMapHeader> header = getMapLoader(stream, name, modName, encoding)->loadMapHeader();
 
 	//might be original campaign and require patch
 	getMapPatcher(name)->patchMapHeader(header);
@@ -86,7 +96,7 @@ std::unique_ptr<CInputStream> CMapService::getStreamFromMem(const ui8 * buffer,
 	return std::unique_ptr<CInputStream>(new CMemoryStream(buffer, size));
 }
 
-std::unique_ptr<IMapLoader> CMapService::getMapLoader(std::unique_ptr<CInputStream> & stream)
+std::unique_ptr<IMapLoader> CMapService::getMapLoader(std::unique_ptr<CInputStream> & stream, std::string mapName, std::string modName, std::string encoding)
 {
 	// Read map header
 	CBinaryReader reader(stream.get());
@@ -109,12 +119,12 @@ std::unique_ptr<IMapLoader> CMapService::getMapLoader(std::unique_ptr<CInputStre
 			// gzip header magic number, reversed for LE
 			case 0x00088B1F:
 				stream = std::unique_ptr<CInputStream>(new CCompressedStream(std::move(stream), true));
-				return std::unique_ptr<IMapLoader>(new CMapLoaderH3M(stream.get()));
+				return std::unique_ptr<IMapLoader>(new CMapLoaderH3M(mapName, modName, encoding, stream.get()));
 			case EMapFormat::WOG :
 			case EMapFormat::AB  :
 			case EMapFormat::ROE :
 			case EMapFormat::SOD :
-				return std::unique_ptr<IMapLoader>(new CMapLoaderH3M(stream.get()));
+				return std::unique_ptr<IMapLoader>(new CMapLoaderH3M(mapName, modName, encoding, stream.get()));
 			default :
 				throw std::runtime_error("Unknown map format");
 		}

lib/mapping/CMapService.h

@@ -58,7 +58,7 @@ public:
 	 * @param name indicates name of file that will be used during map header patching
 	 * @return a unique ptr to the loaded map class
 	 */
-	virtual std::unique_ptr<CMap> loadMap(const ui8 * buffer, int size, const std::string & name) const = 0;
+	virtual std::unique_ptr<CMap> loadMap(const ui8 * buffer, int size, const std::string & name, const std::string & modName, const std::string & encoding) const = 0;
 
 	/**
 	 * Loads the VCMI/H3 map header from a buffer. This method is temporarily
@@ -72,7 +72,7 @@ public:
 	 * @param name indicates name of file that will be used during map header patching
 	 * @return a unique ptr to the loaded map class
 	 */
-	virtual std::unique_ptr<CMapHeader> loadMapHeader(const ui8 * buffer, int size, const std::string & name) const = 0;
+	virtual std::unique_ptr<CMapHeader> loadMapHeader(const ui8 * buffer, int size, const std::string & name, const std::string & modName, const std::string & encoding) const = 0;
 
 	virtual void saveMap(const std::unique_ptr<CMap> & map, boost::filesystem::path fullPath) const = 0;
 };
@@ -85,8 +85,8 @@ public:
 
 	std::unique_ptr<CMap> loadMap(const ResourceID & name) const override;
 	std::unique_ptr<CMapHeader> loadMapHeader(const ResourceID & name) const override;
-	std::unique_ptr<CMap> loadMap(const ui8 * buffer, int size, const std::string & name) const override;
-	std::unique_ptr<CMapHeader> loadMapHeader(const ui8 * buffer, int size, const std::string & name) const override;
+	std::unique_ptr<CMap> loadMap(const ui8 * buffer, int size, const std::string & name, const std::string & modName, const std::string & encoding) const override;
+	std::unique_ptr<CMapHeader> loadMapHeader(const ui8 * buffer, int size, const std::string & name, const std::string & modName, const std::string & encoding) const override;
 	void saveMap(const std::unique_ptr<CMap> & map, boost::filesystem::path fullPath) const override;
 private:
 	/**
@@ -113,7 +113,7 @@ private:
 	 * @param stream the input map stream
 	 * @return the constructed map loader
 	 */
-	static std::unique_ptr<IMapLoader> getMapLoader(std::unique_ptr<CInputStream> & stream);
+	static std::unique_ptr<IMapLoader> getMapLoader(std::unique_ptr<CInputStream> & stream, std::string mapName, std::string modName, std::string encoding);
 
 	/**
 	 * Gets a map patcher for specified scenario

lib/mapping/MapFormatH3M.h

@@ -17,11 +17,11 @@
 
 #include "../int3.h"
 
-#include "../filesystem/CBinaryReader.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 
 class CGHeroInstance;
+class CBinaryReader;
 class CArtifactInstance;
 class CGObjectInstance;
 class CGSeerHut;
@@ -29,6 +29,7 @@ class IQuestObject;
 class CGTownInstance;
 class CCreatureSet;
 class CInputStream;
+class TextIdentifier;
 
 
 class DLL_LINKAGE CMapLoaderH3M : public IMapLoader
@@ -39,7 +40,7 @@ public:
 	 *
 	 * @param stream a stream containing the map data
 	 */
-	CMapLoaderH3M(CInputStream * stream);
+	CMapLoaderH3M(const std::string & mapName, const std::string & modName, const std::string & encodingName, CInputStream * stream);
 
 	/**
 	 * Destructor.
@@ -171,14 +172,14 @@ private:
 	 *
 	 * @return the initialized seer hut object
 	 */
-	CGSeerHut * readSeerHut();
+	CGSeerHut * readSeerHut(const int3 & position);
 
 	/**
 	 * Reads a quest for the given quest guard.
 	 *
 	 * @param guard the quest guard where that quest should be applied to
 	 */
-	void readQuest(IQuestObject * guard);
+	void readQuest(IQuestObject * guard, const int3 & position);
 
 	/**
 	 * Reads a town.
@@ -186,7 +187,7 @@ private:
 	 * @param castleID the id of the castle type
 	 * @return the loaded town object
 	 */
-	CGTownInstance * readTown(int castleID);
+	CGTownInstance * readTown(int castleID, const int3 & position);
 
 	/**
 	 * Converts buildings to the specified castle id.
@@ -206,7 +207,7 @@ private:
 	/**
 	* read optional message and optional guards
 	*/
-	void readMessageAndGuards(std::string& message, CCreatureSet * guards);
+	void readMessageAndGuards(std::string & message, CCreatureSet * guards, const int3 & position);
 
 	void readSpells(std::set<SpellID> & dest);
 
@@ -234,14 +235,13 @@ private:
 	/**
 	* Helper to read map position
 	*/
-	inline int3 readInt3()
-	{
-		int3 p;
-		p.x = reader.readUInt8();
-		p.y = reader.readUInt8();
-		p.z = reader.readUInt8();
-		return p;
-	}
+	int3 readInt3();
+
+	/// reads string from input stream and converts it to unicode
+	std::string readBasicString();
+
+	/// reads string from input stream, converts it to unicode and attempts to translate it
+	std::string readLocalizedString(const TextIdentifier & identifier);
 
 	void afterRead();
 
@@ -257,10 +257,13 @@ private:
 	 * (when loading a map then the mapHeader ptr points to the same object)
 	 */
 	std::unique_ptr<CMapHeader> mapHeader;
-
-	CBinaryReader reader;
+	std::unique_ptr<CBinaryReader> reader;
 	CInputStream * inputStream;
 
+	std::string mapName;
+	std::string modName;
+	std::string fileEncoding;
+
 };
 
 VCMI_LIB_NAMESPACE_END

lib/rmg/ObstaclePlacer.cpp

@@ -23,6 +23,7 @@
 #include "../CRandomGenerator.h"
 #include "Functions.h"
 #include "../mapping/CMapEditManager.h"
+#include "../mapping/CMap.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 

lib/rmg/RmgMap.cpp

@@ -14,6 +14,7 @@
 #include "CMapGenOptions.h"
 #include "Zone.h"
 #include "../mapping/CMapEditManager.h"
+#include "../mapping/CMap.h"
 #include "../CTownHandler.h"
 #include "ObjectManager.h"
 #include "RoadPlacer.h"

lib/rmg/RmgMap.h

@@ -11,10 +11,10 @@
 #pragma once
 #include "../int3.h"
 #include "../GameConstants.h"
-#include "../mapping/CMap.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 
+class CMap;
 class CMapEditManager;
 class TileInfo;
 class CMapGenOptions;

lib/rmg/RockPlacer.cpp

@@ -20,6 +20,7 @@
 #include "../TerrainHandler.h"
 #include "../CRandomGenerator.h"
 #include "../mapping/CMapEditManager.h"
+#include "../mapping/CMap.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 

lib/spells/CSpellHandler.cpp

@@ -694,7 +694,7 @@ CSpell * CSpellHandler::loadFromJson(const std::string & scope, const JsonNode &
 		spell->combat = type == "combat";
 	}
 
-	VLC->generaltexth->registerString(spell->getNameTextID(), json["name"].String());
+	VLC->generaltexth->registerString(scope, spell->getNameTextID(), json["name"].String());
 
 	logMod->trace("%s: loading spell %s", __FUNCTION__, spell->getNameTranslated());
 
@@ -909,7 +909,7 @@ CSpell * CSpellHandler::loadFromJson(const std::string & scope, const JsonNode &
 		const si32 levelPower     = levelObject.power = static_cast<si32>(levelNode["power"].Integer());
 
 		if (!spell->isCreatureAbility())
-			VLC->generaltexth->registerString(spell->getDescriptionTextID(levelIndex), levelNode["description"].String());
+			VLC->generaltexth->registerString(scope, spell->getDescriptionTextID(levelIndex), levelNode["description"].String());
 
 		levelObject.cost          = static_cast<si32>(levelNode["cost"].Integer());
 		levelObject.AIValue       = static_cast<si32>(levelNode["aiValue"].Integer());

mapeditor/mainwindow.cpp

@@ -480,7 +480,7 @@ void MainWindow::addGroupIntoCatalog(const std::string & groupName, bool useCust
 		if(staticOnly && !factory->isStaticObject())
 			continue;
 
-		auto subGroupName = QString::fromStdString(factory->getNameTranslated());
+		auto subGroupName = QString::fromStdString(VLC->objtypeh->getObjectName(ID, secondaryID));
 		
 		auto * itemType = new QStandardItem(subGroupName);
 		for(int templateId = 0; templateId < templates.size(); ++templateId)

server/CGameHandler.cpp

@@ -637,7 +637,7 @@ void CGameHandler::changePrimSkill(const CGHeroInstance * hero, PrimarySkill::Pr
 				InfoWindow iw;
 				iw.player = hero->tempOwner;
 				iw.text.addTxt(MetaString::GENERAL_TXT, 1); //can gain no more XP
-				iw.text.addReplacement(hero->getNameTextID());
+				iw.text.addReplacement(hero->getNameTranslated());
 				sendAndApply(&iw);
 			}
 		}
@@ -860,7 +860,7 @@ void CGameHandler::endBattle(int3 tile, const CGHeroInstance * heroAttacker, con
 		InfoWindow iw;
 		iw.player = finishingBattle->winnerHero->tempOwner;
 		iw.text.addTxt(MetaString::GENERAL_TXT, 221); //Through eagle-eyed observation, %s is able to learn %s
-		iw.text.addReplacement(finishingBattle->winnerHero->getNameTextID());
+		iw.text.addReplacement(finishingBattle->winnerHero->getNameTranslated());
 
 		std::ostringstream names;
 		for (int i = 0; i < cs.spells.size(); i++)
@@ -2826,7 +2826,7 @@ void CGameHandler::useScholarSkill(ObjectInstanceID fromHero, ObjectInstanceID t
 		iw.components.push_back(Component(Component::SEC_SKILL, 18, ScholarSkillLevel, 0));
 
 		iw.text.addTxt(MetaString::GENERAL_TXT, 139);//"%s, who has studied magic extensively,
-		iw.text.addReplacement(h1->getNameTextID());
+		iw.text.addReplacement(h1->getNameTranslated());
 
 		if (!cs2.spells.empty())//if found new spell - apply
 		{
@@ -2844,7 +2844,7 @@ void CGameHandler::useScholarSkill(ObjectInstanceID fromHero, ObjectInstanceID t
 				}
 			}
 			iw.text.addTxt(MetaString::GENERAL_TXT, 142);//from %s
-			iw.text.addReplacement(h2->getNameTextID());
+			iw.text.addReplacement(h2->getNameTranslated());
 			sendAndApply(&cs2);
 		}
 
@@ -2866,9 +2866,10 @@ void CGameHandler::useScholarSkill(ObjectInstanceID fromHero, ObjectInstanceID t
 					case 2:	iw.text.addTxt(MetaString::GENERAL_TXT, 141);
 					case 1:	break;
 					default:	iw.text << ", ";
-				}			}
+				}
+			}
 			iw.text.addTxt(MetaString::GENERAL_TXT, 148);//from %s
-			iw.text.addReplacement(h2->getNameTextID());
+			iw.text.addReplacement(h2->getNameTranslated());
 			sendAndApply(&cs1);
 		}
 		sendAndApply(&iw);