Merge pull request #3730 from IvanSavenko/localization_export

Better export command for translations

client/ClientCommandManager.cpp

@@ -183,44 +183,108 @@ void ClientCommandManager::handleNotDialogCommand()
 	LOCPLINT->showingDialog->setn(false);
 }
 
-void ClientCommandManager::handleConvertTextCommand()
+void ClientCommandManager::handleTranslateGameCommand()
 {
-	logGlobal->info("Searching for available maps");
-	std::unordered_set<ResourcePath> mapList = CResourceHandler::get()->getFilteredFiles([&](const ResourcePath & ident)
+	std::map<std::string, std::map<std::string, std::string>> textsByMod;
+	VLC->generaltexth->exportAllTexts(textsByMod);
+
+	const boost::filesystem::path outPath = VCMIDirs::get().userExtractedPath() / "translation";
+	boost::filesystem::create_directories(outPath);
+
+	for(const auto & modEntry : textsByMod)
 	{
-		return ident.getType() == EResType::MAP;
-	});
+		JsonNode output;
 
-	std::unordered_set<ResourcePath> campaignList = CResourceHandler::get()->getFilteredFiles([&](const ResourcePath & ident)
+		for(const auto & stringEntry : modEntry.second)
+		{
+			if(boost::algorithm::starts_with(stringEntry.first, "map."))
+				continue;
+			if(boost::algorithm::starts_with(stringEntry.first, "campaign."))
+				continue;
+
+			output[stringEntry.first].String() = stringEntry.second;
+		}
+
+		if (!output.isNull())
+		{
+			const boost::filesystem::path filePath = outPath / (modEntry.first + ".json");
+			std::ofstream file(filePath.c_str());
+			file << output.toString();
+		}
+	}
+
+	printCommandMessage("Translation export complete");
+}
+
+void ClientCommandManager::handleTranslateMapsCommand()
+{
+	CMapService mapService;
+
+	printCommandMessage("Searching for available maps");
+	std::unordered_set<ResourcePath> mapList = CResourceHandler::get()->getFilteredFiles([&](const ResourcePath & ident)
 	{
-		return ident.getType() == EResType::CAMPAIGN;
+		return ident.getType() == EResType::MAP;
 	});
 
-	CMapService mapService;
+	std::vector<std::unique_ptr<CMap>> loadedMaps;
+	std::vector<std::shared_ptr<CampaignState>> loadedCampaigns;
 
-	logGlobal->info("Loading maps for export");
+	printCommandMessage("Loading maps for export");
 	for (auto const & mapName : mapList)
 	{
 		try
 		{
 			// load and drop loaded map - we only need loader to run over all maps
-			mapService.loadMap(mapName, nullptr);
+			loadedMaps.push_back(mapService.loadMap(mapName, nullptr));
 		}
 		catch(std::exception & e)
 		{
-			logGlobal->error("Map %s is invalid. Message: %s", mapName.getName(), e.what());
+			logGlobal->warn("Map %s is invalid. Message: %s", mapName.getName(), e.what());
 		}
 	}
 
+	printCommandMessage("Searching for available campaigns");
+	std::unordered_set<ResourcePath> campaignList = CResourceHandler::get()->getFilteredFiles([&](const ResourcePath & ident)
+	{
+		return ident.getType() == EResType::CAMPAIGN;
+	});
+
 	logGlobal->info("Loading campaigns for export");
 	for (auto const & campaignName : campaignList)
 	{
-		auto state = CampaignHandler::getCampaign(campaignName.getName());
-		for (auto const & part : state->allScenarios())
-			state->getMap(part, nullptr);
+		loadedCampaigns.push_back(CampaignHandler::getCampaign(campaignName.getName()));
+		for (auto const & part : loadedCampaigns.back()->allScenarios())
+			loadedCampaigns.back()->getMap(part, nullptr);
 	}
 
-	VLC->generaltexth->dumpAllTexts();
+	std::map<std::string, std::map<std::string, std::string>> textsByMod;
+	VLC->generaltexth->exportAllTexts(textsByMod);
+
+	const boost::filesystem::path outPath = VCMIDirs::get().userExtractedPath() / "translation";
+	boost::filesystem::create_directories(outPath);
+
+	for(const auto & modEntry : textsByMod)
+	{
+		JsonNode output;
+
+		for(const auto & stringEntry : modEntry.second)
+		{
+			if(boost::algorithm::starts_with(stringEntry.first, "map."))
+				output[stringEntry.first].String() = stringEntry.second;
+
+			if(boost::algorithm::starts_with(stringEntry.first, "campaign."))
+				output[stringEntry.first].String() = stringEntry.second;
+		}
+
+		if (!output.isNull())
+		{
+			const boost::filesystem::path filePath = outPath / (modEntry.first + ".json");
+			std::ofstream file(filePath.c_str());
+			file << output.toString();
+		}
+	}
+
+	printCommandMessage("Translation export complete");
 }
 
 void ClientCommandManager::handleGetConfigCommand()
@@ -522,8 +586,11 @@ void ClientCommandManager::processCommand(const std::string & message, bool call
 	else if(commandName == "not dialog")
 		handleNotDialogCommand();
 
-	else if(message=="convert txt")
-		handleConvertTextCommand();
+	else if(message=="translate" || message=="translate game")
+		handleTranslateGameCommand();
+
+	else if(message=="translate maps")
+		handleTranslateMapsCommand();
 
 	else if(message=="get config")
 		handleGetConfigCommand();

client/ClientCommandManager.h

@@ -48,8 +48,11 @@ class ClientCommandManager //take mantis #2292 issue about account if thinking a
 	// Set the state indicating if dialog box is active to "no"
 	void handleNotDialogCommand();
 
-	// Dumps all game text, maps text and campaign maps text into Client log between BEGIN TEXT EXPORT and END TEXT EXPORT
-	void handleConvertTextCommand();
+	// Extracts all translateable game texts into Translation directory, separating files on per-mod basis
+	void handleTranslateGameCommand();
+
+	// Extracts all translateable texts from maps and campaigns into Translation directory, separating files on per-mod basis
+	void handleTranslateMapsCommand();
 
 	// Saves current game configuration into extracted/configuration folder
 	void handleGetConfigCommand();

docs/modders/Translations.md

@@ -38,10 +38,12 @@ VCMI allows translating game data into languages other than English. In order to
 If you have already existing Heroes III translation you can:
 
 - Install VCMI and select your localized Heroes III data files for VCMI data files
-- Launch VCMI_Client.exe directly from game install directory
-- In console window, type `convert txt`
+- Launch VCMI and start any map to get in game
+- Press Tab to activate chat and enter '/translate'
 
-This will export all strings from game into `Documents/My Games/VCMI/VCMI_Client_log.txt` which you can then use to update json files in your translation
+This will export all strings from game into `Documents/My Games/VCMI/extracted/translation/` directory which you can then use to update json files in your translation.
+
+To export maps and campaigns, use '/translate maps' command instead.
 
 ## Translating VCMI data
 
@@ -111,7 +113,15 @@ If everything is OK, your changes will be accepted and will be part of next rele
 
 ### Exporting translation
 
-TODO
+If you want to start new translation for a mod or to update existing one you may need to export it first. To do that:
+
+- Enable mod(s) that you want to export and set game language in Launcher to one that you want to target
+- Launch VCMI and start any map to get in game
+- Press Tab to activate chat and enter '/translate'
+
+After that, start Launcher, switch to Help tab and open "log files directory". You can find exported json's in 'extracted/translation' directory.
+
+If your mod also contains maps or campaigns that you want to translate, then use '/translate maps' command instead.
 
 ### Translating mod information
 In order to display information in Launcher in language selected by user add following block into your mod.json:
@@ -127,6 +137,12 @@ In order to display information in Launcher in language selected by user add fol
 ```
 However, normally you don't need to use block for English. Instead, English text should remain in root section of your mod.json file, to be used when game can not find translated version.
 
+### Tranlating in-game strings
+
+After you have exported translation and added mod information for your language, copy exported file to `<mod directory>/Content/config/<mod name>/<language>.json`.
+
+Use any text editor (Notepad++ is recommended for Windows) and translate all strings from this file to your language
+
 # Developers documentation
 
 ### Adding new languages

docs/players/Cheat_Codes.md

@@ -114,7 +114,8 @@ Below a list of supported commands, with their arguments wrapped in `<>`
 `bonuses` - shows bonuses of currently selected adventure map object
 
 #### Extract commands
-`convert txt` - save game texts into json files  
+`translate` - save game texts into json files  
+`translate maps` - save map and campaign texts into json files  
 `get config` - save game objects data into json files  
 `get scripts` - dumps lua script stuff into files (currently inactive due to scripting disabled for default builds)    
 `get txt` - save game texts into .txt files matching original heroes 3 files  

lib/CBonusTypeHandler.cpp

@@ -239,8 +239,8 @@ void CBonusTypeHandler::loadItem(const JsonNode & source, CBonusType & dest, con
 
 	if (!dest.hidden)
 	{
-		VLC->generaltexth->registerString( "core", dest.getNameTextID(), source["name"].String());
-		VLC->generaltexth->registerString( "core", dest.getDescriptionTextID(), source["description"].String());
+		VLC->generaltexth->registerString( "vcmi", dest.getNameTextID(), source["name"].String());
+		VLC->generaltexth->registerString( "vcmi", dest.getDescriptionTextID(), source["description"].String());
 	}
 
 	const JsonNode & graphics = source["graphics"];

lib/CGeneralTextHandler.cpp

@@ -10,15 +10,16 @@
 #include "StdInc.h"
 #include "CGeneralTextHandler.h"
 
-#include "filesystem/Filesystem.h"
-#include "serializer/JsonSerializeFormat.h"
 #include "CConfigHandler.h"
 #include "GameSettings.h"
-#include "mapObjects/CQuest.h"
-#include "modding/CModHandler.h"
-#include "VCMI_Lib.h"
 #include "Languages.h"
 #include "TextOperations.h"
+#include "VCMIDirs.h"
+#include "VCMI_Lib.h"
+#include "filesystem/Filesystem.h"
+#include "mapObjects/CQuest.h"
+#include "modding/CModHandler.h"
+#include "serializer/JsonSerializeFormat.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
 
@@ -386,18 +387,26 @@ bool TextLocalizationContainer::identifierExists(const TextIdentifier & UID) con
 	return stringsLocalizations.count(UID.get());
 }
 
-void TextLocalizationContainer::dumpAllTexts()
+void TextLocalizationContainer::exportAllTexts(std::map<std::string, std::map<std::string, std::string>> & storage) const
 {
-	logGlobal->info("BEGIN TEXT EXPORT");
-	for(const auto & entry : stringsLocalizations)
+	for (auto const & subContainer : subContainers)
+		subContainer->exportAllTexts(storage);
+
+	for (auto const & entry : stringsLocalizations)
 	{
+		std::string textToWrite;
+		std::string modName = entry.second.modContext;
+
+		if (modName.find('.') != std::string::npos)
+			modName = modName.substr(0, modName.find('.'));
+
 		if (!entry.second.overrideValue.empty())
-			logGlobal->info(R"("%s" : "%s",)", entry.first, TextOperations::escapeString(entry.second.overrideValue));
+			textToWrite = entry.second.overrideValue;
 		else
-			logGlobal->info(R"("%s" : "%s",)", entry.first, TextOperations::escapeString(entry.second.baseValue));
-	}
+			textToWrite = entry.second.baseValue;
 
-	logGlobal->info("END TEXT EXPORT");
+		storage[modName][entry.first] = textToWrite;
+	}
 }
 
 std::string TextLocalizationContainer::getModLanguage(const std::string & modContext)
@@ -491,6 +500,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 	readToVector("core.overview", "DATA/OVERVIEW.TXT" );
 	readToVector("core.arraytxt", "DATA/ARRAYTXT.TXT" );
 	readToVector("core.priskill", "DATA/PRISKILL.TXT" );
+	readToVector("core.plcolors", "DATA/PLCOLORS.TXT" );
 	readToVector("core.jktext",   "DATA/JKTEXT.TXT"   );
 	readToVector("core.tvrninfo", "DATA/TVRNINFO.TXT" );
 	readToVector("core.turndur",  "DATA/TURNDUR.TXT"  );
@@ -545,20 +555,6 @@ CGeneralTextHandler::CGeneralTextHandler():
 		}
 		while (parser.endLine());
 	}
-	{
-		CLegacyConfigParser parser(TextPath::builtin("DATA/PLCOLORS.TXT"));
-		size_t index = 0;
-		do
-		{
-			std::string color = parser.readString();
-
-			registerString("core", {"core.plcolors", index}, color);
-			color[0] = toupper(color[0]);
-			registerString("core", {"vcmi.capitalColors", index}, color);
-			index += 1;
-		}
-		while (parser.endLine());
-	}
 	{
 		CLegacyConfigParser parser(TextPath::builtin("DATA/SEERHUT.TXT"));
 

lib/CGeneralTextHandler.h

@@ -181,8 +181,9 @@ public:
 	/// converts identifier into user-readable string
 	const std::string & deserialize(const TextIdentifier & identifier) const;
 	
-	/// Debug method, dumps all currently known texts into console using Json-like format
-	void dumpAllTexts();
+	/// Debug method, returns all currently stored texts
+	/// Format: [mod ID][string ID] -> human-readable text
+	void exportAllTexts(std::map<std::string, std::map<std::string, std::string>> & storage) const;
 	
 	/// Add or override subcontainer which can store identifiers
 	void addSubContainer(const TextLocalizationContainer & container);

lib/mapObjects/CQuest.cpp

@@ -684,6 +684,9 @@ void CGSeerHut::serializeJsonOptions(JsonSerializeFormat & handler)
 		//backward compatibility for VCMI maps that use old SeerHut format
 		auto s = handler.enterStruct("reward");
 		const JsonNode & rewardsJson = handler.getCurrent();
+
+		if (rewardsJson.Struct().empty())
+			return;
 		
 		std::string fullIdentifier;
 		std::string metaTypeName;