khanat-opennel-code/code/ryzom/tools/translation_tools/extract_new_sheet_names.cpp

// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010  Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

#include "nel/misc/types_nl.h"
#include "nel/misc/config_file.h"
#include "nel/misc/sheet_id.h"
#include "nel/misc/path.h"
#include "nel/misc/diff_tool.h"
#include "nel/misc/algo.h"
#include "nel/georges/u_form.h"
#include "nel/georges/u_form_elm.h"
#include "nel/georges/load_form.h"
#include "nel/ligo/ligo_config.h"
#include "nel/ligo/primitive.h"
#include "nel/ligo/primitive_utils.h"

using namespace std;
using namespace NLMISC;
using namespace NLLIGO;
using namespace STRING_MANAGER;


static CLigoConfig		LigoConfig;
static bool	RemoveOlds = false;


// ***************************************************************************
/*
 *	Interface to build the whole list of words (key id) for a specific worksheet
 */
struct IWordListBuilder
{
	virtual bool	buildWordList(std::vector<string> &allWords, string workSheetFileName) =0;
};


// ***************************************************************************
/*
 *	Specialisation of IWordListBuilder to list sheets in a directory
 */
struct CSheetWordListBuilder : public IWordListBuilder
{
	string		SheetExt;
	string		SheetPath;

	virtual bool	buildWordList(std::vector<string> &allWords, string workSheetFileName)
	{
		SheetExt= toLower(SheetExt);

		// verify the directory is correct
		if(!CFile::isDirectory(SheetPath))
		{
			nlwarning("Error: Directory '%s' not found. '%s' Aborted", SheetPath.c_str(), workSheetFileName.c_str());
			return false;
		}

		// list all files.
		std::vector<string>		allFiles;
		allFiles.reserve(100000);
		CPath::getPathContent(SheetPath, true, false, true, allFiles, NULL);

		// Keep only the extension we want, and remove "_" (parent)
		allWords.clear();
		allWords.reserve(allFiles.size());
		for(uint i=0;i<allFiles.size();i++)
		{
			string	fileNameWithoutExt= CFile::getFilenameWithoutExtension(allFiles[i]);
			string	extension= toLower(CFile::getExtension(allFiles[i]));
			// bad extension?
			if(extension!=SheetExt)
				continue;
			// parent?
			if(fileNameWithoutExt.empty()||fileNameWithoutExt[0]=='_')
				continue;
			// ok, add
			allWords.push_back(toLower(fileNameWithoutExt));
		}

		return true;
	}

};


// ***************************************************************************
/*
 *	Specialisation of IWordListBuilder to list new region/place name from .primitive
 */
struct CRegionPrimWordListBuilder : public IWordListBuilder
{
	string			PrimPath;
	vector<string>	PrimFilter;

	virtual bool	buildWordList(std::vector<string> &allWords, string workSheetFileName)
	{
		// verify the directory is correct
		if(!CFile::isDirectory(PrimPath))
		{
			nlwarning("Error: Directory '%s' not found. '%s' Aborted", PrimPath.c_str(), workSheetFileName.c_str());
			return false;
		}

		// list all files.
		std::vector<string>		allFiles;
		allFiles.reserve(100000);
		CPath::getPathContent(PrimPath, true, false, true, allFiles, NULL);

		// parse all primitive that match the filter
		allWords.clear();
		allWords.reserve(100000);
		// to avoid duplicate
		set<string>		allWordSet;
		for(uint i=0;i<allFiles.size();i++)
		{
			string	fileName= CFile::getFilename(allFiles[i]);
			// filter don't match?
			bool	oneMatch= false;
			for(uint filter=0;filter<PrimFilter.size();filter++)
			{
				if(testWildCard(fileName, PrimFilter[filter]))
					oneMatch= true;
			}
			if(!oneMatch)
				continue;

			// ok, read the file
			CPrimitives PrimDoc;
			CPrimitiveContext::instance().CurrentPrimitive = &PrimDoc;
			if (!loadXmlPrimitiveFile(PrimDoc, allFiles[i], LigoConfig))
			{
				nlwarning("Error: cannot open file '%s'. '%s' Aborted", allFiles[i].c_str(), workSheetFileName.c_str());
				CPrimitiveContext::instance().CurrentPrimitive = NULL;
				return false;
			}
			CPrimitiveContext::instance().CurrentPrimitive = NULL;

			// For all primitives of interest
			const char	*listClass[]= {"continent", "region", "place", "stable",
				"teleport_destination", "room_template"};
			const char	*listProp[]= {"name", "name", "name", "name",
				"place_name", "place_name"};
			const uint	numListClass= sizeof(listClass)/sizeof(listClass[0]);
			const uint	numListProp= sizeof(listProp)/sizeof(listProp[0]);
			nlctassert(numListProp==numListClass);
			for(uint cid=0;cid<numListClass;cid++)
			{
				// parse the whole hierarchy
				TPrimitiveClassPredicate predCont(listClass[cid]);
				CPrimitiveSet<TPrimitiveClassPredicate> setPlace;
				TPrimitiveSet placeRes;
				setPlace.buildSet(PrimDoc.RootNode, predCont, placeRes);
				// for all found
				for (uint placeId= 0; placeId < placeRes.size(); ++placeId)
				{
					string primName;
					if(placeRes[placeId]->getPropertyByName(listProp[cid], primName) && !primName.empty())
					{
						primName= toLower(primName);
						// avoid duplicate
						if(allWordSet.insert(primName).second)
						{
							allWords.push_back(primName);
						}
					}
				}
			}
		}

		return true;
	}
};


// ***************************************************************************
void	extractNewWords(string workSheetFileName, string columnId, IWordListBuilder &wordListBuilder)
{
	uint	i;

	// **** Load the excel sheet
	// load
	TWorksheet		workSheet;
	if(!loadExcelSheet(workSheetFileName, workSheet, true))
	{
		nlwarning("Error reading '%s'. Aborted", workSheetFileName.c_str());
		return;
	}
	// get the key column index
	uint	keyColIndex = 0;

	if(!workSheet.findCol(columnId, keyColIndex))
	{
		nlwarning("Error: Don't find the column '%s'. '%s' Aborted", columnId.c_str(), workSheetFileName.c_str());
		return;
	}
	// get the name column index
	uint	nameColIndex = 0;
	if(!workSheet.findCol(ucstring("name"), nameColIndex))
	{
		nlwarning("Error: Don't find the column 'name'. '%s' Aborted", workSheetFileName.c_str());
		return;
	}
	// Make a copy of this worksheet, with strlwr on the key
	// Yoyo: I prefer not modify the original worksheet (don't know what bad side effect it can have....)
	TWorksheet		workSheetLwr= workSheet;
	for(i=0;i<workSheetLwr.size();i++)
	{
		ucstring	key= workSheetLwr.getData(i, keyColIndex);
		workSheetLwr.setData(i, keyColIndex, toLower(key));
	}


	// **** List all words with the builder given
	std::vector<string>		allWords;
	if(!wordListBuilder.buildWordList(allWords, workSheetFileName))
		return;


	// **** Append new one to the worksheet
	uint	nbAdd= 0;
	for(i=0;i<allWords.size();i++)
	{
		string	keyName= allWords[i];
		uint rowIdx;
		// search in the key lowred worksheet (avoid case bugs (they do exist...))
		if (!workSheetLwr.findRow(keyColIndex, keyName, rowIdx))
		{
			// we need to add the entry. Add it to the 2 workSheet to maintain coherence (avoid non unique etc...)
			rowIdx = workSheetLwr.size();
			// add to the workSheetLwr
			workSheetLwr.resize(workSheetLwr.size()+1);
			workSheetLwr.setData(rowIdx, keyColIndex, keyName);
			workSheetLwr.setData(rowIdx, nameColIndex, string("<GEN>")+keyName);
			// add to the workSheet
			workSheet.resize(workSheet.size()+1);
			workSheet.setData(rowIdx, keyColIndex, keyName);
			workSheet.setData(rowIdx, nameColIndex, string("<GEN>")+keyName);

			nbAdd++;
		}
	}


	// **** Remove no more present ones (and log)
	uint	nbRemove= 0;
	if(RemoveOlds)
	{
		// Build as a set
		std::set<string>	allWordSet;
		for(i=0;i<allWords.size();i++)
			allWordSet.insert(allWords[i]);
		// For all rows, append to a copy if not erased
		TWorksheet	tmpCopy, tmpCopyLwr;
		nlassert(workSheet.ColCount==workSheetLwr.ColCount);
		nlassert(workSheet.size()==workSheetLwr.size());
		tmpCopy.setColCount(workSheet.ColCount);
		tmpCopy.resize(workSheet.size());
		tmpCopyLwr.setColCount(workSheet.ColCount);
		tmpCopyLwr.resize(workSheet.size());
		uint	dstRowId=0;
		for(i=0;i<workSheet.size();i++)
		{
			string	keyStr= workSheetLwr.getData(i, keyColIndex).toString();
			// if first line, or if the key (lwred) is found in the list of files
			if(i==0 || allWordSet.find(keyStr)!=allWordSet.end())
			{
				tmpCopy.Data[dstRowId]= workSheet.Data[i];
				tmpCopyLwr.Data[dstRowId]= workSheetLwr.Data[i];
				dstRowId++;
			}
			else
			{
				nbRemove++;
				// log
				NLMISC::InfoLog->displayRawNL("'%s': '%s' entry erased at line '%d'.", workSheetFileName.c_str(),
					keyStr.c_str(), i);
			}
		}
		// resize to correct new size
		tmpCopy.resize(dstRowId);
		tmpCopyLwr.resize(dstRowId);

		// copy back
		workSheet= tmpCopy;
		workSheetLwr= tmpCopyLwr;
	}


	// **** Save
	if(nbAdd==0 && nbRemove==0)
	{
		if(RemoveOlds)
			NLMISC::InfoLog->displayRawNL("'%s': No deprecated entry found.", workSheetFileName.c_str());
		NLMISC::InfoLog->displayRawNL("'%s': No new entry found.", workSheetFileName.c_str());
		// Don't save
	}
	else
	{
		if(RemoveOlds)
			NLMISC::InfoLog->displayRawNL("'%s': %d deprecated entry erased.", workSheetFileName.c_str(), nbRemove);
		NLMISC::InfoLog->displayRawNL("'%s': %d new entry found.", workSheetFileName.c_str(), nbAdd);
		// Save the not lowered worksheet
		ucstring s = prepareExcelSheet(workSheet);
		try
		{
			CI18N::writeTextFile(workSheetFileName.c_str(), s, false);
		}
		catch (const Exception &e)
		{
			nlwarning("cannot save file: '%s'. Reason: %s", workSheetFileName.c_str(), e.what());
		}
	}
}


// ***************************************************************************
int extractNewSheetNames(int argc, char *argv[])
{
	// **** read the parameters
	for (int i=2; i<argc; ++i)
	{
		string s = argv[i];
		if (s == "-r")
		{
			// active remove mode
			RemoveOlds = true;
		}
		else
		{
			nlwarning("Unknow option '%s'", argv[i]);
			return -1;
		}
	}

	// **** avoid some flood
	NLMISC::createDebug();
	NLMISC::DebugLog->addNegativeFilter("numCol changed to");
	NLMISC::InfoLog->addNegativeFilter("CPath::addSearchPath");


	// **** read the configuration file
	CConfigFile	cf;
	cf.load("bin/translation_tools.cfg");
	CConfigFile::CVar &paths = cf.getVar("Paths");
	CConfigFile::CVar &pathNoRecurse= cf.getVar("PathsNoRecurse");
	CConfigFile::CVar &ligoClassFile= cf.getVar("LigoClassFile");
	CConfigFile::CVar &leveldesignDataPathVar = cf.getVar("LeveldesignDataPath");

	// parse path
	for (uint i=0; i<paths.size(); ++i)
	{
		CPath::addSearchPath(NLMISC::expandEnvironmentVariables(paths.asString(i)), true, false);
	}
	for (uint i=0; i<pathNoRecurse.size(); ++i)
	{
		CPath::addSearchPath(NLMISC::expandEnvironmentVariables(pathNoRecurse.asString(i)), false, false);
	}

	std::string leveldesignDataPath = CPath::standardizePath(NLMISC::expandEnvironmentVariables(leveldesignDataPathVar.asString()));

	// init ligo config once
	string ligoPath = CPath::lookup(NLMISC::expandEnvironmentVariables(ligoClassFile.asString()), true, true);
	LigoConfig.readPrimitiveClass(ligoPath.c_str(), false);
	NLLIGO::Register();
	CPrimitiveContext::instance().CurrentLigoConfig = &LigoConfig;

	// **** Parse all the different type of sheets
	const char	*sheetDefs[]=
	{
		// 1st is the name of the worksheet file.
		// 2nd is the Key column identifier.
		// 3rd is the sheet extension
		// 4th is the directory where to find new sheets
		"work/item_words_wk.txt",		"item ID",		"sitem",		"leveldesign/game_element/sitem",
		"work/creature_words_wk.txt",	"creature ID",	"creature",		"leveldesign/game_elem/creature/fauna",	// take fauna only because other are special
		"work/sbrick_words_wk.txt",		"sbrick ID",	"sbrick",		"leveldesign/game_element/sbrick",
		"work/sphrase_words_wk.txt",	"sphrase ID",	"sphrase",		"leveldesign/game_element/sphrase",
	};
	uint	numSheetDefs= sizeof(sheetDefs) / (4*sizeof(sheetDefs[0]));

	// For all different type of sheet
	for(uint i=0;i<numSheetDefs;i++)
	{
		CSheetWordListBuilder	builder;
		builder.SheetExt= sheetDefs[i*4+2];
		builder.SheetPath= leveldesignDataPath + sheetDefs[i*4+3];
		extractNewWords(sheetDefs[i*4+0], sheetDefs[i*4+1], builder);
	}

	// **** Parse place and region names
	{
		// build place names
		CRegionPrimWordListBuilder	builder;
		builder.PrimPath= leveldesignDataPath + "primitives";
		builder.PrimFilter.push_back("region_*.primitive");
		builder.PrimFilter.push_back("indoors_*.primitive");
		extractNewWords("work/place_words_wk.txt", "placeId", builder);
	}

	return 0;
}