Khaganat-NeL/khanat-opennel-code
2
0
Fork 0
Code Issues 21 Pull requests Projects Releases 1 Packages Wiki Activity

Merge with develop

Browse source
This commit is contained in:
kervala 2016-02-02 12:37:59 +01:00
parent 058b36aa4a
commit 4808dfcecb
15 changed files with 699 additions and 561 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
code/nel/include/nel/misc/common.h
View file

@ -336,6 +336,10 @@ void itoaInt64 (sint64 number, char *str, sint64 base = 10);
std::string bytesToHumanReadable (const std::string &bytes);
std::string bytesToHumanReadable (uint64 bytes);
/// Convert a number in bytes into a string that is easily readable by an human, for example 105123 -> "102kb"
/// Using units array as string: 0 => B, 1 => KiB, 2 => MiB, 3 => GiB, etc...
std::string bytesToHumanReadableUnits (uint64 bytes, const std::vector<std::string> &units);
/// Convert a human readable into a bytes, for example "102kb" -> 105123
uint32 humanReadableToBytes (const std::string &str);

29
code/nel/include/nel/misc/string_common.h
View file

@ -246,27 +246,38 @@ inline bool fromString(const std::string &str, bool &val)
{
if (str.length() == 1)
{
if (str[0] == '1')
const char c = str[0];
switch(c)
{
case '1':
case 't':
case 'T':
case 'y':
case 'Y':
val = true;
}
else if (str[0] == '0')
{
break;
case '0':
case 'f':
case 'F':
case 'n':
case 'N':
val = false;
}
else
{
break;
default:
val = false;
return false;
}
}
else
{
if (str == "true")
if (str == "true" || str == "yes")
{
val = true;
}
else if (str == "false")
else if (str == "false" || str == "no")
{
val = false;
}

50
code/nel/src/misc/common.cpp
View file

@ -380,7 +380,26 @@ string bytesToHumanReadable (uint64 bytes)
div++;
res = newres;
}
return toString ("%" NL_I64 "u%s", res, divTable[div]);
return toString ("%" NL_I64 "u %s", res, divTable[div]);
}
std::string bytesToHumanReadableUnits (uint64 bytes, const std::vector<std::string> &units)
{
if (units.empty()) return "";
uint div = 0;
uint last = units.size()-1;
uint64 res = bytes;
uint64 newres = res;
for(;;)
{
newres /= 1024;
if(newres < 8 || div > 3 || div == last)
break;
++div;
res = newres;
}
return toString ("%" NL_I64 "u %s", res, units[div].c_str());
}
uint32 humanReadableToBytes (const string &str)
@ -394,7 +413,8 @@ uint32 humanReadableToBytes (const string &str)
if(str[0]<'0' || str[0]>'9')
return 0;
res = atoi (str.c_str());
if (!fromString(str, res))
return 0;
if(str[str.size()-1] == 'B')
{
@ -404,10 +424,28 @@ uint32 humanReadableToBytes (const string &str)
// there's no break and it's **normal**
switch (str[str.size()-2])
{
case 'G': res *= 1024;
case 'M': res *= 1024;
case 'K': res *= 1024;
default: ;
// kB/KB, MB, GB and TB are 1000 multiples
case 'T': res *= 1000;
case 'G': res *= 1000;
case 'M': res *= 1000;
case 'k': res *= 1000; break; // kilo symbol should be a lowercase K
case 'K': res *= 1000; break;
case 'i':
{
// KiB, MiB, GiB and TiB are 1024 multiples
if (str.size()<4)
return res;
switch (str[str.size()-3])
{
case 'T': res *= 1024;
case 'G': res *= 1024;
case 'M': res *= 1024;
case 'K': res *= 1024;
default: ;
}
}
default: ;
}
}

14
code/nel/tools/nel_unit_test/nel_unit_test.cpp
View file

@ -114,6 +114,20 @@ int main(int argc, char *argv[])
// init Nel context
new NLMISC::CApplicationContext;
// disable nldebug messages in logs in Release
#ifdef NL_RELEASE
NLMISC::DisableNLDebug = true;
#endif
NLMISC::createDebug(NULL);
#ifndef NL_DEBUG
NLMISC::INelContext::getInstance().getDebugLog()->removeDisplayer("DEFAULT_SD");
NLMISC::INelContext::getInstance().getInfoLog()->removeDisplayer("DEFAULT_SD");
NLMISC::INelContext::getInstance().getWarningLog()->removeDisplayer("DEFAULT_SD");
NLMISC::INelContext::getInstance().getErrorLog()->removeDisplayer("DEFAULT_SD");
#endif // NL_DEBUG
bool noerrors = false;
try

2
code/nel/tools/nel_unit_test/ut_misc.h
View file

@ -19,6 +19,7 @@
#include "ut_misc_co_task.h"
#include "ut_misc_command.h"
#include "ut_misc_common.h"
#include "ut_misc_config_file.h"
#include "ut_misc_debug.h"
#include "ut_misc_dynlibload.h"
@ -38,6 +39,7 @@ struct CUTMisc : public Test::Suite
{
add(auto_ptr<Test::Suite>(new CUTMiscCoTask));
add(auto_ptr<Test::Suite>(new CUTMiscCommand));
add(auto_ptr<Test::Suite>(new CUTMiscCommon));
add(auto_ptr<Test::Suite>(new CUTMiscConfigFile));
add(auto_ptr<Test::Suite>(new CUTMiscDebug));
add(auto_ptr<Test::Suite>(new CUTMiscDynLibLoad));

171
code/nel/tools/nel_unit_test/ut_misc_common.h Normal file
View file

@ -0,0 +1,171 @@
// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef UT_MISC_COMMON
#define UT_MISC_COMMON
#include <limits>
#include <nel/misc/common.h>
struct CUTMiscCommon : public Test::Suite
{
CUTMiscCommon()
{
TEST_ADD(CUTMiscCommon::bytesToHumanReadableUnits);
TEST_ADD(CUTMiscCommon::humanReadableToBytes);
// Add a line here when adding a new test METHOD
}
void bytesToHumanReadableUnits()
{
std::vector<std::string> units;
std::string res;
// no unit, returns an empty string
res = NLMISC::bytesToHumanReadableUnits(0, units);
TEST_ASSERT(res.empty());
// support bytes
units.push_back("B");
// 0 bytes
res = NLMISC::bytesToHumanReadableUnits(0, units);
TEST_ASSERT(res == "0 B");
// 1000 bytes in B
res = NLMISC::bytesToHumanReadableUnits(1000, units);
TEST_ASSERT(res == "1000 B");
// 1024 bytes in B
res = NLMISC::bytesToHumanReadableUnits(1024, units);
TEST_ASSERT(res == "1024 B");
// support kibibytes
units.push_back("KiB");
// 1000 bytes in B or KiB
res = NLMISC::bytesToHumanReadableUnits(1000, units);
TEST_ASSERT(res == "1000 B");
// 1024 bytes in B or KiB
res = NLMISC::bytesToHumanReadableUnits(1024, units);
TEST_ASSERT(res == "1024 B");
// 1 MB in B or KiB
res = NLMISC::bytesToHumanReadableUnits(1000 * 1000, units);
TEST_ASSERT(res == "976 KiB");
// 1 MiB in B or KiB
res = NLMISC::bytesToHumanReadableUnits(1024 * 1024, units);
TEST_ASSERT(res == "1024 KiB");
// 1 GB in B or KiB
res = NLMISC::bytesToHumanReadableUnits(1000 * 1000 * 1000, units);
TEST_ASSERT(res == "976562 KiB");
// 1 GiB in B or KiB
res = NLMISC::bytesToHumanReadableUnits(1024 * 1024 * 1024, units);
TEST_ASSERT(res == "1048576 KiB");
// support mebibytes
units.push_back("MiB");
// 1 GB in B, KiB or MiB
res = NLMISC::bytesToHumanReadableUnits(1000 * 1000 * 1000, units);
TEST_ASSERT(res == "953 MiB");
// 1 GiB in B, KiB or MiB
res = NLMISC::bytesToHumanReadableUnits(1024 * 1024 * 1024, units);
TEST_ASSERT(res == "1024 MiB");
}
void humanReadableToBytes()
{
uint64 bytes = 0;
// kiB is a wrong unit
bytes = NLMISC::humanReadableToBytes("1kiB");
TEST_ASSERT(bytes == 1);
// 1 kibibyte
bytes = NLMISC::humanReadableToBytes("1KiB");
TEST_ASSERT(bytes == 1024);
// 1 mebibyte
bytes = NLMISC::humanReadableToBytes("1MiB");
TEST_ASSERT(bytes == 1024*1024);
// 1 kilobyte
bytes = NLMISC::humanReadableToBytes("1KB");
TEST_ASSERT(bytes == 1000);
// 1 kilobyte
bytes = NLMISC::humanReadableToBytes("1kB");
TEST_ASSERT(bytes == 1000);
// 1 megabyte
bytes = NLMISC::humanReadableToBytes("1MB");
TEST_ASSERT(bytes == 1000*1000);
// 1 byte
bytes = NLMISC::humanReadableToBytes("1B");
TEST_ASSERT(bytes == 1);
// 1 byte
bytes = NLMISC::humanReadableToBytes("1");
TEST_ASSERT(bytes == 1);
// kiB is a wrong unit
bytes = NLMISC::humanReadableToBytes("1 kiB");
TEST_ASSERT(bytes == 1);
// 1 kibibyte
bytes = NLMISC::humanReadableToBytes("1 KiB");
TEST_ASSERT(bytes == 1024);
// 1 mebibyte
bytes = NLMISC::humanReadableToBytes("1 MiB");
TEST_ASSERT(bytes == 1024*1024);
// 1 kilobyte
bytes = NLMISC::humanReadableToBytes("1 KB");
TEST_ASSERT(bytes == 1000);
// 1 kilobyte
bytes = NLMISC::humanReadableToBytes("1 kB");
TEST_ASSERT(bytes == 1000);
// 1 megabyte
bytes = NLMISC::humanReadableToBytes("1 MB");
TEST_ASSERT(bytes == 1000*1000);
// 1 byte
bytes = NLMISC::humanReadableToBytes("1 B");
TEST_ASSERT(bytes == 1);
// not a number
bytes = NLMISC::humanReadableToBytes("AB");
TEST_ASSERT(bytes == 0);
// not a positive number
bytes = NLMISC::humanReadableToBytes("-1 B");
TEST_ASSERT(bytes == 0);
}
};
#endif

81
code/nel/tools/nel_unit_test/ut_misc_string_common.h
View file

@ -682,60 +682,59 @@ struct CUTMiscStringCommon : public Test::Suite
// tests for bool
bool val;
// true value
val = false;
// true values
ret = NLMISC::fromString("1", val);
TEST_ASSERT(val);
TEST_ASSERT_MSG(ret, "should succeed");
TEST_ASSERT(ret && val);
val = false;
NLMISC::fromString("t", val);
TEST_ASSERT(val);
ret = NLMISC::fromString("t", val);
TEST_ASSERT(ret && val);
val = false;
NLMISC::fromString("y", val);
TEST_ASSERT(val);
ret = NLMISC::fromString("y", val);
TEST_ASSERT(ret && val);
val = false;
NLMISC::fromString("T", val);
TEST_ASSERT(val);
ret = NLMISC::fromString("T", val);
TEST_ASSERT(ret && val);
val = false;
NLMISC::fromString("Y", val);
TEST_ASSERT(val);
ret = NLMISC::fromString("Y", val);
TEST_ASSERT(ret && val);
val = true;
ret = NLMISC::fromString("true", val);
TEST_ASSERT(ret && val);
ret = NLMISC::fromString("yes", val);
TEST_ASSERT(ret && val);
// false values
ret = NLMISC::fromString("0", val);
TEST_ASSERT(!val);
TEST_ASSERT_MSG(ret, "should succeed");
TEST_ASSERT(ret && !val);
val = true;
NLMISC::fromString("f", val);
TEST_ASSERT(!val);
ret = NLMISC::fromString("f", val);
TEST_ASSERT(ret && !val);
val = true;
NLMISC::fromString("n", val);
TEST_ASSERT(!val);
ret = NLMISC::fromString("n", val);
TEST_ASSERT(ret && !val);
val = true;
NLMISC::fromString("F", val);
TEST_ASSERT(!val);
ret = NLMISC::fromString("F", val);
TEST_ASSERT(ret && !val);
val = true;
NLMISC::fromString("N", val);
TEST_ASSERT(!val);
ret = NLMISC::fromString("N", val);
TEST_ASSERT(ret && !val);
ret = NLMISC::fromString("false", val);
TEST_ASSERT(ret && !val);
ret = NLMISC::fromString("no", val);
TEST_ASSERT(ret && !val);
// wrong values
ret = NLMISC::fromString("YES", val);
TEST_ASSERT(!ret && !val);
ret = NLMISC::fromString("foo", val);
TEST_ASSERT(!ret && !val);
// bad character
ret = NLMISC::fromString("a", val);
TEST_ASSERT_MSG(!ret, "should not succeed");
val = true;
NLMISC::fromString("a", val);
TEST_ASSERT_MSG(val, "should not modify the value");
val = false;
NLMISC::fromString("a", val);
TEST_ASSERT_MSG(!val, "should not modify the value");
TEST_ASSERT(!ret && !val);
}
};

2
code/ryzom/client/src/init.cpp
View file

@ -659,7 +659,7 @@ static void addPaths(IProgressCallback &progress, const std::vector<std::string>
std::vector<std::string> directoryPrefixes;
// current directory has priority everywhere
directoryPrefixes.push_back("");
directoryPrefixes.push_back(CPath::standardizePath(CPath::getCurrentPath()));
#if defined(NL_OS_WINDOWS)
// check in same directory as executable

7
code/ryzom/client/src/login_patch.cpp
View file

@ -54,6 +54,7 @@
#include "login.h"
#include "user_agent.h"
#include "seven_zip/seven_zip.h"
#ifndef RY_BG_DOWNLOADER
#include "client_cfg.h"
@ -2625,7 +2626,7 @@ void CPatchThread::processFile (CPatchManager::SFileToPatch &rFTP)
// try to unpack the file
try
{
if (!CPatchManager::unpackLZMA(pPM->ClientPatchPath+lzmaFile, OutFilename+".tmp"))
if (!unpackLZMA(pPM->ClientPatchPath+lzmaFile, OutFilename+".tmp"))
{
// fallback to standard patch method
usePatchFile = true;
@ -3087,7 +3088,7 @@ bool CPatchManager::download(const std::string& patchFullname, const std::string
&& patchName.substr(patchName.size() - zsStrLength) == zsStr)
{
std::string outFilename = patchName.substr(0, patchName.size() - zsStrLength);
CPatchManager::unpack7Zip(patchName, outFilename);
unpack7Zip(patchName, outFilename);
pPM->deleteFile(patchName);
pPM->renameFile(outFilename, sourceFullname);
}
@ -3449,7 +3450,7 @@ void CInstallThread::run()
std::string outFilename = patchName.substr(0, patchName.size() - zsStrLength);
std::string localOutFilename = CPath::standardizeDosPath(outFilename);
if ( CPatchManager::unpackLZMA(patchName, localOutFilename) )
if ( unpackLZMA(patchName, localOutFilename) )
{
pPM->deleteFile(patchName);
pPM->renameFile(outFilename, sourceName);

4
code/ryzom/client/src/login_patch.h
View file

@ -331,10 +331,6 @@ private:
void clearDataScanLog();
static void getCorruptedFileInfo(const SFileToPatch &ftp, ucstring &sTranslate);
// utility func to decompress a monofile 7zip archive
static bool unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName);
// utility func to decompress a single LZMA packed file
static bool unpackLZMA(const std::string &sevenZipFile, const std::string &destFileName);
static bool downloadAndUnpack(const std::string& patchPath, const std::string& sourceFilename, const std::string& extractPath, const std::string& tmpDirectory, uint32 timestamp);
// Forward message to Installation Software
void onFileInstallFinished();

468
code/ryzom/client/src/login_patch_seven_zip.cpp
View file

@ -1,468 +0,0 @@
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// Includes
//
#include "stdpch.h"
#include "login_patch.h"
#define RZ_USE_SEVENZIP 1
#define RZ_USE_NEW_LZMA
#ifdef RZ_USE_SEVENZIP
#ifdef RZ_USE_NEW_LZMA
#include "seven_zip/7z.h"
#include "seven_zip/7zAlloc.h"
#include "seven_zip/7zBuf.h"
#include "seven_zip/7zCrc.h"
#include "seven_zip/7zFile.h"
#include "seven_zip/7zVersion.h"
#include "seven_zip/LzmaDec.h"
#else
#include "seven_zip/7zCrc.h"
#include "seven_zip/7zIn.h"
#include "seven_zip/7zExtract.h"
#include "seven_zip/LzmaDecode.h"
#endif
//
// Namespaces
//
using namespace std;
using namespace NLMISC;
#ifdef RZ_USE_NEW_LZMA
/// Input stream class for 7zip archive
class CNel7ZipInStream : public ISeekInStream
{
NLMISC::IStream *_Stream;
public:
/// Constructor, only allow file stream because 7zip will 'seek' in the stream
CNel7ZipInStream(NLMISC::IStream *s)
: _Stream(s)
{
Read = readFunc;
Seek = seekFunc;
}
// the read function called by 7zip to read data
static SRes readFunc(void *object, void *buffer, size_t *size)
{
try
{
CNel7ZipInStream *me = (CNel7ZipInStream*)object;
uint len = (uint)*size;
me->_Stream->serialBuffer((uint8*)buffer, len);
return SZ_OK;
}
catch (...)
{
return SZ_ERROR_READ;
}
}
// the seek function called by seven zip to seek inside stream
static SRes seekFunc(void *object, Int64 *pos, ESzSeek origin)
{
try
{
CNel7ZipInStream *me = (CNel7ZipInStream*)object;
sint32 offset = (sint32)*pos;
bool ret = me->_Stream->seek(offset, (NLMISC::IStream::TSeekOrigin)origin);
if (ret)
{
*pos = (Int64)me->_Stream->getPos();
return SZ_OK;
}
}
catch (...)
{
}
return SZ_ERROR_READ;
}
};
#else
/// Input stream class for 7zip archive
class CNel7ZipInStream : public _ISzInStream
{
NLMISC::IStream *_Stream;
public:
/// Constructor, only allow file stream because 7zip will 'seek' in the stream
CNel7ZipInStream(NLMISC::IStream *s)
: _Stream(s)
{
Read = readFunc;
Seek = seekFunc;
}
// the read function called by 7zip to read data
static SZ_RESULT readFunc(void *object, void *buffer, size_t size, size_t *processedSize)
{
try
{
CNel7ZipInStream *me = (CNel7ZipInStream*)object;
me->_Stream->serialBuffer((uint8*)buffer, (uint)size);
*processedSize = size;
return SZ_OK;
}
catch (...)
{
return SZE_FAIL;
}
}
// the seek function called by seven zip to seek inside stream
static SZ_RESULT seekFunc(void *object, CFileSize pos)
{
try
{
CNel7ZipInStream *me = (CNel7ZipInStream*)object;
bool ret= me->_Stream->seek(pos, NLMISC::IStream::begin);
if (ret)
return SZ_OK;
else
return SZE_FAIL;
}
catch (...)
{
return SZE_FAIL;
}
}
};
#endif
#endif
bool CPatchManager::unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName)
{
#ifdef RZ_USE_SEVENZIP
nlinfo("Uncompressing 7zip archive '%s' to '%s'", sevenZipFile.c_str(), destFileName.c_str());
// init seven zip
ISzAlloc allocImp;
allocImp.Alloc = SzAlloc;
allocImp.Free = SzFree;
ISzAlloc allocTempImp;
allocTempImp.Alloc = SzAllocTemp;
allocTempImp.Free = SzFreeTemp;
// wrap file in a CIFile
CIFile input(sevenZipFile);
CNel7ZipInStream inStr(&input);
#ifdef RZ_USE_NEW_LZMA
CLookToRead lookStream;
lookStream.realStream = &inStr;
LookToRead_CreateVTable(&lookStream, False);
LookToRead_Init(&lookStream);
CrcGenerateTable();
CSzArEx db;
SzArEx_Init(&db);
// unpack the file using the 7zip API
SRes res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp);
if (res != SZ_OK)
{
nlerror("Failed to open archive file %s", sevenZipFile.c_str());
return false;
}
if (db.NumFiles != 1)
{
nlerror("Seven zip archive with more than 1 file are unsupported");
return false;
}
UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */
Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */
size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */
size_t offset;
size_t outSizeProcessed = 0;
// get the first file
res = SzArEx_Extract(&db, &lookStream.s, 0,
&blockIndex, &outBuffer, &outBufferSize,
&offset, &outSizeProcessed,
&allocImp, &allocTempImp);
// get the length of first file
size_t nameLen = SzArEx_GetFileNameUtf16(&db, 0, NULL);
ucstring filename;
filename.resize(nameLen);
// write filename into ucstring
SzArEx_GetFileNameUtf16(&db, 0, &filename[0]);
// write the extracted file
FILE *outputHandle = fopen(destFileName.c_str(), "wb+");
if (outputHandle == 0)
{
nlerror("Can not open output file '%s'", destFileName.c_str());
return false;
}
UInt32 processedSize = (UInt32)fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle);
if (processedSize != outSizeProcessed)
{
nlerror("Failed to write %u char to output file '%s'", outSizeProcessed-processedSize, destFileName.c_str());
return false;
}
fclose(outputHandle);
IAlloc_Free(&allocImp, outBuffer);
// free 7z context
SzArEx_Free(&db, &allocImp);
#else
InitCrcTable();
CArchiveDatabaseEx db;
SzArDbExInit(&db);
// unpack the file using the 7zip API
SZ_RESULT res = SzArchiveOpen(&inStr, &db, &allocImp, &allocTempImp);
if (res != SZ_OK)
{
nlerror("Failed to open archive file %s", sevenZipFile.c_str());
return false;
}
if (db.Database.NumFiles != 1)
{
nlerror("Seven zip archive with more than 1 file are unsupported");
return false;
}
UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */
Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */
size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */
size_t offset = 0;
size_t outSizeProcessed = 0;
// get the first file
res = SzExtract(&inStr, &db, 0,
&blockIndex, &outBuffer, &outBufferSize,
&offset, &outSizeProcessed,
&allocImp, &allocTempImp);
// write the extracted file
FILE *outputHandle = fopen(destFileName.c_str(), "wb+");
if (outputHandle == 0)
{
nlerror("Can not open output file '%s'", destFileName.c_str());
return false;
}
UInt32 processedSize = (UInt32)fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle);
if (processedSize != outSizeProcessed)
{
nlerror("Failed to write %u char to output file '%s'", outSizeProcessed-processedSize, destFileName.c_str());
return false;
}
fclose(outputHandle);
allocImp.Free(outBuffer);
// free 7z context
SzArDbExFree(&db, allocImp.Free);
#endif
// ok, all is fine, file is unpacked
return true;
#else
return false;
#endif
}
bool CPatchManager::unpackLZMA(const std::string &lzmaFile, const std::string &destFileName)
{
#ifdef RZ_USE_SEVENZIP
nldebug("unpackLZMA : decompression the lzma file '%s' into output file '%s", lzmaFile.c_str(), destFileName.c_str());
CIFile inStream(lzmaFile);
uint32 inSize = inStream.getFileSize();
#ifdef RZ_USE_NEW_LZMA
if (inSize < LZMA_PROPS_SIZE + 8)
{
nlwarning("Invalid file size, too small file '%s'", lzmaFile.c_str());
return false;
}
// read the compressed file in buffer
auto_ptr<uint8> inBuffer = auto_ptr<uint8>(new uint8[inSize]);
inStream.serialBuffer(inBuffer.get(), inSize);
uint8 *pos = inBuffer.get();
pos += LZMA_PROPS_SIZE;
// read the output file size
size_t fileSize = 0;
for (int i = 0; i < 8; ++i)
{
if (pos >= inBuffer.get()+inSize)
{
nlassert(false);
return false;
}
fileSize |= ((UInt64)*pos++) << (8 * i);
}
// allocators
ISzAlloc allocImp;
allocImp.Alloc = SzAlloc;
allocImp.Free = SzFree;
CLzmaDec state;
LzmaDec_Construct(&state);
// allocate and decode props and probs
SRes res = LzmaDec_Allocate(&state, inBuffer.get(), LZMA_PROPS_SIZE, &allocImp);
if (res != 0)
{
nlwarning("Failed to decode lzma properies in file '%s'!", lzmaFile.c_str());
return false;
}
// allocate the output buffer
auto_ptr<uint8> outBuffer = auto_ptr<uint8>(new uint8[fileSize]);
// in and out file sizes
SizeT outProcessed = fileSize;
SizeT inProcessed = (SizeT)(inSize-(pos-inBuffer.get()));
LzmaDec_Init(&state);
// decompress the file in memory
ELzmaStatus status;
res = LzmaDec_DecodeToBuf(&state, (Byte*)outBuffer.get(), &outProcessed, (Byte*)pos, &inProcessed, LZMA_FINISH_ANY, &status);
// free memory
LzmaDec_Free(&state, &allocImp);
if (res != 0 || outProcessed != fileSize)
{
nlwarning("Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), (sint)status);
return false;
}
#else
auto_ptr<uint8> inBuffer = auto_ptr<uint8>(new uint8[inSize]);
inStream.serialBuffer(inBuffer.get(), inSize);
CLzmaDecoderState state;
uint8 *pos = inBuffer.get();
// read the lzma properties
int res = LzmaDecodeProperties(&state.Properties, (unsigned char*) pos, LZMA_PROPERTIES_SIZE);
if (res != 0)
{
nlwarning("Failed to decode lzma properies in file '%s'!", lzmaFile.c_str());
return false;
}
if (inSize < LZMA_PROPERTIES_SIZE + 8)
{
nlwarning("Invalid file size, too small file '%s'", lzmaFile.c_str());
return false;
}
// alloc the probs, making sure they are deleted in function exit
size_t nbProb = LzmaGetNumProbs(&state.Properties);
auto_ptr<CProb> probs = auto_ptr<CProb>(new CProb[nbProb]);
state.Probs = probs.get();
pos += LZMA_PROPERTIES_SIZE;
// read the output file size
size_t fileSize = 0;
for (int i = 0; i < 8; i++)
{
if (pos >= inBuffer.get()+inSize)
{
nlassert(false);
return false;
}
fileSize |= ((UInt64)*pos++) << (8 * i);
}
SizeT outProcessed = 0;
SizeT inProcessed = 0;
// allocate the output buffer
auto_ptr<uint8> outBuffer = auto_ptr<uint8>(new uint8[fileSize]);
// decompress the file in memory
res = LzmaDecode(&state, (unsigned char*) pos, (SizeT)(inSize-(pos-inBuffer.get())), &inProcessed, (unsigned char*)outBuffer.get(), (SizeT)fileSize, &outProcessed);
if (res != 0 || outProcessed != fileSize)
{
nlwarning("Failed to decode lzma file '%s'", lzmaFile.c_str());
return false;
}
#endif
// store on output buffer
COFile outStream(destFileName);
outStream.serialBuffer(outBuffer.get(), (uint)fileSize);
return true;
#else
return false;
#endif
}

357
code/ryzom/client/src/seven_zip/seven_zip.cpp Normal file
View file

@ -0,0 +1,357 @@
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#include "seven_zip.h"
#include "nel/misc/types_nl.h"
#include "nel/misc/file.h"
#include "nel/misc/path.h"
#include "7z.h"
#include "7zAlloc.h"
#include "7zCrc.h"
#include "7zVersion.h"
#include "LzmaLib.h"
#include <memory>
//
// Namespaces
//
using namespace std;
using namespace NLMISC;
/// Input stream class for 7zip archive
class CNel7ZipInStream : public ISeekInStream
{
NLMISC::IStream *_Stream;
public:
/// Constructor, only allow file stream because 7zip will 'seek' in the stream
CNel7ZipInStream(NLMISC::IStream *s)
: _Stream(s)
{
Read = readFunc;
Seek = seekFunc;
}
// the read function called by 7zip to read data
static SRes readFunc(void *object, void *buffer, size_t *size)
{
try
{
CNel7ZipInStream *me = (CNel7ZipInStream*)object;
uint len = (uint)*size;
me->_Stream->serialBuffer((uint8*)buffer, len);
return SZ_OK;
}
catch (...)
{
return SZ_ERROR_READ;
}
}
// the seek function called by seven zip to seek inside stream
static SRes seekFunc(void *object, Int64 *pos, ESzSeek origin)
{
try
{
CNel7ZipInStream *me = (CNel7ZipInStream*)object;
sint32 offset = (sint32)*pos;
bool ret = me->_Stream->seek(offset, (NLMISC::IStream::TSeekOrigin)origin);
if (ret)
{
*pos = (Int64)me->_Stream->getPos();
return SZ_OK;
}
}
catch (...)
{
}
return SZ_ERROR_READ;
}
};
bool unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName)
{
nlinfo("Uncompressing 7zip archive '%s' to '%s'", sevenZipFile.c_str(), destFileName.c_str());
// init seven zip
ISzAlloc allocImp;
allocImp.Alloc = SzAlloc;
allocImp.Free = SzFree;
ISzAlloc allocTempImp;
allocTempImp.Alloc = SzAllocTemp;
allocTempImp.Free = SzFreeTemp;
// wrap file in a CIFile
CIFile input(sevenZipFile);
CNel7ZipInStream inStr(&input);
CLookToRead lookStream;
lookStream.realStream = &inStr;
LookToRead_CreateVTable(&lookStream, False);
LookToRead_Init(&lookStream);
CrcGenerateTable();
CSzArEx db;
SzArEx_Init(&db);
// unpack the file using the 7zip API
SRes res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp);
if (res != SZ_OK)
{
nlerror("Failed to open archive file %s", sevenZipFile.c_str());
return false;
}
if (db.NumFiles != 1)
{
nlerror("Seven zip archive with more than 1 file are unsupported");
return false;
}
UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */
Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */
size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */
size_t offset;
size_t outSizeProcessed = 0;
// get the first file
res = SzArEx_Extract(&db, &lookStream.s, 0,
&blockIndex, &outBuffer, &outBufferSize,
&offset, &outSizeProcessed,
&allocImp, &allocTempImp);
// get the length of first file
size_t nameLen = SzArEx_GetFileNameUtf16(&db, 0, NULL);
ucstring filename;
filename.resize(nameLen);
// write filename into ucstring
SzArEx_GetFileNameUtf16(&db, 0, &filename[0]);
// write the extracted file
FILE *outputHandle = fopen(destFileName.c_str(), "wb+");
if (outputHandle == 0)
{
nlerror("Can not open output file '%s'", destFileName.c_str());
return false;
}
UInt32 processedSize = (UInt32)fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle);
if (processedSize != outSizeProcessed)
{
nlerror("Failed to write %u char to output file '%s'", outSizeProcessed-processedSize, destFileName.c_str());
return false;
}
fclose(outputHandle);
IAlloc_Free(&allocImp, outBuffer);
// free 7z context
SzArEx_Free(&db, &allocImp);
// ok, all is fine, file is unpacked
return true;
}
bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName)
{
nldebug("unpackLZMA: decompress LZMA file '%s' to '%s", lzmaFile.c_str(), destFileName.c_str());
// open input file
CIFile inStream(lzmaFile);
uint32 inSize = inStream.getFileSize();
if (inSize < LZMA_PROPS_SIZE + 8)
{
nlwarning("unpackLZMA: Invalid file size, too small file '%s'", lzmaFile.c_str());
return false;
}
// allocate input buffer for props
auto_ptr<uint8> propsBuffer = auto_ptr<uint8>(new uint8[LZMA_PROPS_SIZE]);
// size of LZMA content
inSize -= LZMA_PROPS_SIZE + 8;
// allocate input buffer for lzma data
auto_ptr<uint8> inBuffer = auto_ptr<uint8>(new uint8[inSize]);
uint64 fileSize = 0;
try
{
// read props
inStream.serialBuffer(propsBuffer.get(), LZMA_PROPS_SIZE);
// read uncompressed size
inStream.serial(fileSize);
// read lzma content
inStream.serialBuffer(inBuffer.get(), inSize);
}
catch(const EReadError &e)
{
nlwarning("unpackLZMA: Error while reading '%s': %s", lzmaFile.c_str(), e.what());
return false;
}
// allocate the output buffer
auto_ptr<uint8> outBuffer = auto_ptr<uint8>(new uint8[fileSize]);
// in and out file sizes
SizeT outProcessed = (SizeT)fileSize;
SizeT inProcessed = (SizeT)inSize;
// decompress the file in memory
sint res = LzmaUncompress(outBuffer.get(), &outProcessed, inBuffer.get(), &inProcessed, propsBuffer.get(), LZMA_PROPS_SIZE);
if (res != 0 || outProcessed != fileSize)
{
nlwarning("unpackLZMA: Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), res);
return false;
}
// store on output buffer
COFile outStream(destFileName);
try
{
// write content
outStream.serialBuffer(outBuffer.get(), (uint)fileSize);
}
catch(const EFile &e)
{
nlwarning("unpackLZMA: Error while writing '%s': %s", destFileName.c_str(), e.what());
CFile::deleteFile(destFileName);
return false;
}
return true;
}
bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName)
{
nldebug("unpackLZMA: compress '%s' to LZMA file '%s", srcFileName.c_str(), lzmaFileName.c_str());
// open file
CIFile inStream(srcFileName);
size_t inSize = inStream.getFileSize();
// file empty
if (!inSize)
{
nlwarning("packLZMA: File '%s' not found or empty", srcFileName.c_str());
return false;
}
// allocate input buffer
auto_ptr<uint8> inBuffer = auto_ptr<uint8>(new uint8[inSize]);
try
{
// read file in buffer
inStream.serialBuffer(inBuffer.get(), inSize);
}
catch(const EReadError &e)
{
nlwarning("packLZMA: Error while reading '%s': %s", srcFileName.c_str(), e.what());
return false;
}
// allocate output buffer
size_t outSize = (11 * inSize / 10) + 65536; // worst case = 1.1 * size + 64K
auto_ptr<uint8> outBuffer = auto_ptr<uint8>(new uint8[outSize]);
// allocate buffer for props
size_t outPropsSize = LZMA_PROPS_SIZE;
auto_ptr<uint8> outProps = auto_ptr<uint8>(new uint8[outPropsSize]);
// compress with best compression and other default settings
sint res = LzmaCompress(outBuffer.get(), &outSize, inBuffer.get(), inSize, outProps.get(), &outPropsSize, 9, 1 << 24, 3, 0, 2, 32, 1);
switch(res)
{
case SZ_OK:
{
// store on output buffer
COFile outStream(lzmaFileName);
// unable to create file
if (!outStream.isOpen())
{
nlwarning("packLZMA: Unable to create '%s'", srcFileName.c_str());
return false;
}
try
{
// write props
outStream.serialBuffer(outProps.get(), (uint)outPropsSize);
// write uncompressed size
uint64 uncompressSize = inSize;
outStream.serial(uncompressSize);
// write content
outStream.serialBuffer(outBuffer.get(), (uint)outSize);
}
catch(const EFile &e)
{
nlwarning("packLZMA: Error while writing '%s': %s", lzmaFileName.c_str(), e.what());
CFile::deleteFile(lzmaFileName);
return false;
}
return true;
}
case SZ_ERROR_MEM:
nlwarning("packLZMA: Memory allocation error while compressing '%s' (input buffer size: %u, output buffer size: %u)", srcFileName.c_str(), (uint)inSize, (uint)outSize);
break;
case SZ_ERROR_PARAM:
nlwarning("packLZMA: Incorrect parameter while compressing '%s'", srcFileName.c_str());
break;
case SZ_ERROR_OUTPUT_EOF:
nlwarning("packLZMA: Output buffer overflow while compressing '%s' (input buffer size: %u, output buffer size: %u)", srcFileName.c_str(), (uint)inSize, (uint)outSize);
break;
case SZ_ERROR_THREAD:
nlwarning("packLZMA: Errors in multithreading functions (only for Mt version)");
break;
default:
nlwarning("packLZMA: Unknown error (%d) while compressing '%s'", res, srcFileName.c_str());
}
return false;
}

31
code/ryzom/client/src/seven_zip/seven_zip.h Normal file
View file

@ -0,0 +1,31 @@
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef SEVEN_ZIP_H
#define SEVEN_ZIP_H
#include <string>
// utility func to decompress a monofile 7zip archive
bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFileName);
// utility func to decompress a single LZMA packed file
bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName);
// utility func to compress a single file to LZMA packed file
bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName);
#endif

6
code/ryzom/tools/patch_gen/CMakeLists.txt
View file

@ -1,13 +1,15 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip)
SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h)
SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h)
ADD_EXECUTABLE(patch_gen ${MAIN_SRC})
TARGET_LINK_LIBRARIES(patch_gen ryzom_gameshare nelmisc nelnet nelligo nelgeorges)
TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges)
NL_DEFAULT_PROPS(patch_gen "Ryzom, Tools: Patch Generator")
NL_ADD_RUNTIME_FLAGS(patch_gen)
ADD_EXECUTABLE(patch_gen_service WIN32 ${SERVICE_SRC})
TARGET_LINK_LIBRARIES(patch_gen_service ryzom_gameshare nelmisc nelnet nelligo nelgeorges)
TARGET_LINK_LIBRARIES(patch_gen_service ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges)
NL_DEFAULT_PROPS(patch_gen_service "Ryzom, Tools: Patch Generator Service")
NL_ADD_RUNTIME_FLAGS(patch_gen_service)

34
code/ryzom/tools/patch_gen/patch_gen_common.cpp
View file

@ -27,6 +27,7 @@
#include "nel/misc/command.h"
#include "nel/misc/sstring.h"
#include "game_share/singleton_registry.h"
#include "seven_zip.h"
using namespace std;
using namespace NLMISC;
@ -74,37 +75,16 @@ void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,c
#endif // NL_OS_WINDOWS
}
void GenerateLZMA(const std::string sourceFile, const std::string &outputFile)
void GenerateLZMA(const std::string &sourceFile, const std::string &outputFile)
{
{
// old syntax incompatible with new versions
std::string cmd = "lzma e";
cmd += " " + sourceFile + " " + outputFile;
nlinfo("Executing system command: %s", cmd.c_str());
nlinfo("Compressing %s to %s using LZMA...", sourceFile.c_str(), outputFile.c_str());
}
#ifdef NL_OS_WINDOWS
_spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL);
#else // NL_OS_WINDOWS
// new lzma only supports one file name on command line, so make a copy
CFile::copyFile(outputFile, sourceFile);
// new lzma syntax, -z = compress, -9 = best compression
std::string cmd = NLMISC::toString("lzma -z -9 %s", outputFile.c_str());
sint error = system(cmd.c_str());
if (error)
if (!packLZMA(sourceFile, outputFile))
{
nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
CFile::deleteFile(outputFile);
nlwarning("LZMA compress failed");
}
else
{
// lzma always append a .lzma extension, so rename compressed file to wanted one
CFile::moveFile(outputFile, outputFile + ".lzma");
}
#endif // NL_OS_WINDOWS
}
@ -230,7 +210,7 @@ void CPackageDescription::storeToPdr(CPersistentDataRecord& pdr) const
void CPackageDescription::readIndex(CBNPFileSet& packageIndex) const
{
std::string indexPath = _RootDirectory + _IndexFileName;
nlinfo("Reading history file: %s ...", indexPath.c_str());
// clear out old contents before reading from input file
@ -294,7 +274,7 @@ void CPackageDescription::generateClientIndex(CProductDescriptionForClient& theC
std::string patchNumber = toString("%05u", packageIndex.getVersionNumber());
std::string patchDirectory = _PatchDirectory + patchNumber;
std::string patchFile = patchDirectory + "/" + _ClientIndexFileName;
nlinfo("Generating client index: %s...", patchFile.c_str());
// make sure the version sub directory exist