Khaganat-NeL/khanat-opennel-code
2
0
Fork 0
Code Issues 21 Pull requests Projects Releases 1 Packages Wiki Activity

Fixed: Translations files are all using UTF-8 by default now (and LF instead of CRLF), fixes #128

Browse source 
--HG--
branch : develop
This commit is contained in:
kervala 2016-10-29 19:50:28 +02:00
parent 7f8b9954f5
commit 1c872d9ef9
7 changed files with 41 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
code/nel/include/nel/misc/diff_tool.h
View file

@ -21,7 +21,7 @@
namespace STRING_MANAGER
{
const ucstring nl("\r\n");
const ucstring nl("\n");
struct TStringInfo

5
code/nel/include/nel/misc/i18n.h
View file

@ -162,11 +162,8 @@ public:
* EF,BB, BF.
* 16 bits encoding can be recognized by the official header :
* FF, FE, witch can be reversed if the data are MSB first.
*
* Optionally, you can force the reader to consider the file as
* UTF-8 encoded.
*/
static void readTextBuffer(uint8 *buffer, uint size, ucstring &result, bool forceUtf8 = false);
static void readTextBuffer(uint8 *buffer, uint size, ucstring &result);
/** Remove any C style comment from the passed string.
*/

14
code/nel/src/misc/diff_tool.cpp
View file

@ -117,7 +117,7 @@ bool loadStringFile(const std::string filename, vector<TStringInfo> &stringInfos
*/
ucstring text;
CI18N::readTextFile(filename, text, false, false, true, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, text, false, false, true, CI18N::LINE_FMT_LF);
// CI18N::readTextBuffer(buffer, size, text);
// delete [] buffer;
@ -313,7 +313,7 @@ bool readPhraseFile(const std::string &filename, vector<TPhrase> &phrases, bool
{
ucstring doc;
CI18N::readTextFile(filename, doc, false, false, true, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, doc, false, false, true, CI18N::LINE_FMT_LF);
return readPhraseFileFromString(doc, filename, phrases, forceRehash);
}
@ -577,7 +577,7 @@ ucstring preparePhraseFile(const vector<TPhrase> &phrases, bool removeDiffCommen
if (!c.Comments.empty())
{
ucstring comment = tabLines(1, c.Comments);
ret += comment; // + '\r'+'\n';
ret += comment; // + '\n';
}
if (!c.Conditions.empty())
{
@ -626,7 +626,7 @@ bool loadExcelSheet(const string filename, TWorksheet &worksheet, bool checkUniq
fp.close();
ucstring str;
CI18N::readTextFile(filename, str, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, str, false, false, false, CI18N::LINE_FMT_LF);
if (!readExcelSheet(str, worksheet, checkUnique))
return false;
@ -646,6 +646,8 @@ bool readExcelSheet(const ucstring &str, TWorksheet &worksheet, bool checkUnique
strArray[strArray.size()-1]= 0;
memcpy(&strArray[0], &str[0], str.size()*sizeof(ucchar));
// size of new line characters
size_t sizeOfNl = nl.length();
// **** Build array of lines. just point to strArray, and fill 0 where appropriated
vector<ucchar*> lines;
@ -660,10 +662,10 @@ bool readExcelSheet(const ucstring &str, TWorksheet &worksheet, bool checkUnique
// nldebug("Found line : [%s]", ucstring(&strArray[lastPos]).toString().c_str());
lines.push_back(&strArray[lastPos]);
}
lastPos = pos + 2;
lastPos = pos + sizeOfNl;
}
// Must add last line if no \r\n ending
// Must add last line if no \n ending
if (lastPos < str.size())
{
pos= str.size();

26
code/nel/src/misc/i18n.cpp
View file

@ -743,7 +743,7 @@ void CI18N::_readTextFile(const string &filename,
// Transform the string in ucstring according to format header
if (!text.empty())
readTextBuffer((uint8*)&text[0], (uint)text.size(), result, forceUtf8);
readTextBuffer((uint8*)&text[0], (uint)text.size(), result);
if (preprocess)
{
@ -1137,28 +1137,13 @@ void CI18N::_readTextFile(const string &filename,
}
}
void CI18N::readTextBuffer(uint8 *buffer, uint size, ucstring &result, bool forceUtf8)
void CI18N::readTextBuffer(uint8 *buffer, uint size, ucstring &result)
{
static uint8 utf16Header[] = { 0xffu, 0xfeu };
static uint8 utf16RevHeader[] = { 0xfeu, 0xffu };
static uint8 utf8Header[] = { 0xefu, 0xbbu, 0xbfu };
if (forceUtf8)
{
if (size>=3 &&
buffer[0]==utf8Header[0] &&
buffer[1]==utf8Header[1] &&
buffer[2]==utf8Header[2]
)
{
// remove utf8 header
buffer+= 3;
size-=3;
}
string text((char*)buffer, size);
result.fromUtf8(text);
}
else if (size>=3 &&
if (size>=3 &&
buffer[0]==utf8Header[0] &&
buffer[1]==utf8Header[1] &&
buffer[2]==utf8Header[2]
@ -1211,10 +1196,9 @@ void CI18N::readTextBuffer(uint8 *buffer, uint size, ucstring &result, bool forc
}
else
{
// hum.. ascii read ?
// so, just do a direct conversion
// all text files without BOM are now parsed as UTF-8 by default
string text((char*)buffer, size);
result = text;
result.fromUtf8(text);
}
}

6
code/ryzom/tools/translation_tools/extract_bot_names.cpp
View file

@ -745,11 +745,11 @@ int extractBotNames(int argc, char *argv[])
// saving the modified files
ucstring s = prepareExcelSheet(botNames);
CI18N::writeTextFile(workBotNamesFile.asString(), s, false);
CI18N::writeTextFile(workBotNamesFile.asString(), s);
s = prepareExcelSheet(transBotNames);
CI18N::writeTextFile(transBotNamesFile.asString(), s, false);
CI18N::writeTextFile(transBotNamesFile.asString(), s);
s = prepareExcelSheet(fcts);
CI18N::writeTextFile(workTitleFile.asString(), s, false);
CI18N::writeTextFile(workTitleFile.asString(), s);
return 0;
}

2
code/ryzom/tools/translation_tools/extract_new_sheet_names.cpp
View file

@ -315,7 +315,7 @@ void extractNewWords(string workSheetFileName, string columnId, IWordListBuilder
ucstring s = prepareExcelSheet(workSheet);
try
{
CI18N::writeTextFile(workSheetFileName.c_str(), s, false);
CI18N::writeTextFile(workSheetFileName.c_str(), s);
}
catch (const Exception &e)
{

44
code/ryzom/tools/translation_tools/main.cpp
View file

@ -190,7 +190,7 @@ bool readPhraseFile1(const std::string &filename, vector<TPhrase> &phrases, bool
{
ucstring doc;
CI18N::readTextFile(filename, doc, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, doc, false, false, false, CI18N::LINE_FMT_LF);
verifyVersion(doc, 1);
return readPhraseFileFromString(doc, filename, phrases, forceRehash);
}
@ -199,7 +199,7 @@ bool readPhraseFile2(const std::string &filename, vector<TPhrase> &phrases, bool
{
ucstring doc;
CI18N::readTextFile(filename, doc, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, doc, false, false, false, CI18N::LINE_FMT_LF);
verifyVersion(doc, 2);
return readPhraseFileFromString(doc, filename, phrases, forceRehash);
}
@ -352,7 +352,7 @@ bool mergeStringDiff(vector<TStringInfo> &strings, const string &language, const
{
// Check if the diff is translated
ucstring text;
CI18N::readTextFile(diffs[i], text, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(diffs[i], text, false, false, false, CI18N::LINE_FMT_LF);
if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
{
LOG("Diff file [%s] is not translated, merging it later.\n", CFile::getFilename(diffs[i]).c_str());
@ -679,7 +679,7 @@ void cleanComment(const std::string & filename)
ucstring text;
uint nbOldValue=0;
CI18N::readTextFile(filename, text, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, text, false, false, false, CI18N::LINE_FMT_LF);
ucstring newText;
ucstring::size_type last = 0;
@ -800,7 +800,7 @@ int mergeStringDiff(int argc, char *argv[])
{
// backup the original file
ucstring old;
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_LF);
if (old != str)
CFile::moveFile(historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename), filename);
}
@ -840,7 +840,7 @@ bool mergePhraseDiff(vector<TPhrase> &phrases, const string &language, bool only
{
// Check if the diff is translated
ucstring text;
CI18N::readTextFile(diffs[i], text, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(diffs[i], text, false, false, false, CI18N::LINE_FMT_LF);
verifyVersion(text, 1);
if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
{
@ -1164,7 +1164,7 @@ int mergePhraseDiff(int argc, char *argv[], int version)
{
// backup the original file
ucstring old;
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_LF);
if (old != str)
CFile::moveFile(historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename), filename);
}
@ -1333,7 +1333,7 @@ int mergeClauseDiff(int argc, char *argv[])
{
// backup the original file
ucstring old;
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_LF);
if (old != str)
CFile::moveFile(historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename), filename);
}
@ -1358,7 +1358,7 @@ bool mergeWorksheetDiff(const std::string filename, TWorksheet &sheet, bool only
if (onlyTranslated)
{
ucstring text;
CI18N::readTextFile(fileList[i], text, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(fileList[i], text, false, false, false, CI18N::LINE_FMT_LF);
if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
{
LOG("Diff file [%s] is not translated, merging it later.\n", CFile::getFilename(fileList[i]).c_str());
@ -1666,7 +1666,7 @@ int makeWorksheetDiff(int argc, char *argv[], const std::string &additionFilenam
string fn(CFile::getFilenameWithoutExtension(referenceFilename)), ext(CFile::getExtension(referenceFilename));
std::string diffName(diffDir+fn+"_diff_"+diffVersion+"."+ext);
CI18N::writeTextFile(diffName, str, false);
CI18N::writeTextFile(diffName, str);
}
@ -1691,9 +1691,9 @@ int mergeWorksheetDiff(int argc, char *argv[], const std::string &filename, cons
// there is no translated file yet, build one from the working file.
ucstring str;
string addfn = addDir+additionFile;
CI18N::readTextFile(addfn, str, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(addfn, str, false, false, false, CI18N::LINE_FMT_LF);
str = str.substr(0, str.find(nl)+2);
CI18N::writeTextFile(transDir+filename, str, false);
CI18N::writeTextFile(transDir+filename, str);
// reread the file.
bool res = loadExcelSheet(transDir+filename, translated);
nlassert(res);
@ -1711,7 +1711,7 @@ int mergeWorksheetDiff(int argc, char *argv[], const std::string &filename, cons
{
// backup the original file
ucstring old;
CI18N::readTextFile(transDir+filename, old, false, true, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(transDir+filename, old, false, true, false, CI18N::LINE_FMT_LF);
if (old != str)
{
string fn(CFile::getFilenameWithoutExtension(filename)), ext(CFile::getExtension(filename));
@ -1720,7 +1720,7 @@ int mergeWorksheetDiff(int argc, char *argv[], const std::string &filename, cons
}
if (translated.size() > 0)
CI18N::writeTextFile(transDir+filename, str, false);
CI18N::writeTextFile(transDir+filename, str);
return 0;
}
@ -1865,7 +1865,7 @@ void cropLines(const std::string &filename, uint32 nbLines)
LOG("Cropping %u lines from file '%s'\n", nbLines, filename.c_str());
CI18N::readTextFile(filename, utext, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, utext, false, false, false, CI18N::LINE_FMT_LF);
string text = utext.toUtf8();
@ -1881,7 +1881,7 @@ void cropLines(const std::string &filename, uint32 nbLines)
utext.fromUtf8(text);
CI18N::writeTextFile(filename, utext, true);
CI18N::writeTextFile(filename, utext);
}
@ -1910,7 +1910,7 @@ int makeWork()
// change #include "*_en.txt" into #include "*_wk.txt"
ucstring utext;
CI18N::readTextFile(filename, utext, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, utext, false, false, false, CI18N::LINE_FMT_LF);
string text = utext.toUtf8();
bool changedFile = false;
@ -1943,7 +1943,7 @@ int makeWork()
if (changedFile)
{
utext.fromUtf8(text);
CI18N::writeTextFile(filename, utext, true);
CI18N::writeTextFile(filename, utext);
}
// change filename
@ -2348,7 +2348,7 @@ void patchWorkFile(vector<TPhrase> &updatedPhrase, const std::string & filename)
{
ucstring text;
if ( updatedPhrase.empty() ) { return; }
CI18N::readTextFile(filename, text, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, text, false, false, false, CI18N::LINE_FMT_LF);
vector<TPhrase>::const_iterator first(updatedPhrase.begin());
vector<TPhrase>::const_iterator last(updatedPhrase.end());
for (; first != last; ++first)
@ -2489,7 +2489,7 @@ bool mergePhraseDiff2(vector<TPhrase> &phrases, const string &language, bool onl
{
// Check if the diff is translated
ucstring text;
CI18N::readTextFile(diffs[i], text, false, false, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(diffs[i], text, false, false, false, CI18N::LINE_FMT_LF);
verifyVersion(text, 2);
if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
{
@ -2947,7 +2947,7 @@ void preprocessTextFile(const std::string &filename,
// Transform the string in ucstring according to format header
if (!text.empty())
CI18N::readTextBuffer((uint8*)&text[0], (uint)text.size(), result, false);
CI18N::readTextBuffer((uint8*)&text[0], (uint)text.size(), result);
@ -3029,7 +3029,7 @@ int mergePhraseDiff(int argc, char *argv[])
{
// backup the original file
ucstring old;
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_CRLF);
CI18N::readTextFile(filename, old, false, true, false, CI18N::LINE_FMT_LF);
if (old != str)
CFile::moveFile((historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename)).c_str(), filename.c_str());
}